1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 if (UseCompressedClassPointers) {
2237 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmpw rscratch1, r10");
2240 } else {
2241 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2242 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2243 st->print_cr("\tcmp rscratch1, r10");
2244 }
2245 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2246 }
2247 #endif
2248
2249 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2250 {
2251 __ ic_check(InteriorEntryAlignment);
2252 }
2253
2254 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2255 {
2256 return MachNode::size(ra_);
2257 }
2258
2259 // REQUIRED EMIT CODE
2260
2261 //=============================================================================
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 address base = __ start_a_stub(size_deopt_handler());
2269 if (base == nullptr) {
2270 ciEnv::current()->record_failure("CodeCache is full");
2271 return 0; // CodeBuffer::expand failed
2272 }
2273
2274 int offset = __ offset();
2275 Label start;
2276 __ bind(start);
2277 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 int entry_offset = __ offset();
2280 __ b(start);
2281
2282 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2283 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2284 "out of bounds read in post-call NOP check");
2285 __ end_a_stub();
2286 return entry_offset;
2287 }
2288
2289 // REQUIRED MATCHER CODE
2290
2291 //=============================================================================
2292
2293 bool Matcher::match_rule_supported(int opcode) {
2294 if (!has_match_rule(opcode))
2295 return false;
2296
2297 switch (opcode) {
2298 case Op_OnSpinWait:
2299 return VM_Version::supports_on_spin_wait();
2300 case Op_CacheWB:
2301 case Op_CacheWBPreSync:
2302 case Op_CacheWBPostSync:
2303 if (!VM_Version::supports_data_cache_line_flush()) {
2304 return false;
2305 }
2306 break;
2307 case Op_ExpandBits:
2308 case Op_CompressBits:
2309 if (!VM_Version::supports_svebitperm()) {
2310 return false;
2311 }
2312 break;
2313 case Op_FmaF:
2314 case Op_FmaD:
2315 case Op_FmaVF:
2316 case Op_FmaVD:
2317 if (!UseFMA) {
2318 return false;
2319 }
2320 break;
2321 case Op_FmaHF:
2322 // UseFMA flag also needs to be checked along with FEAT_FP16
2323 if (!UseFMA || !is_feat_fp16_supported()) {
2324 return false;
2325 }
2326 break;
2327 case Op_AddHF:
2328 case Op_SubHF:
2329 case Op_MulHF:
2330 case Op_DivHF:
2331 case Op_MinHF:
2332 case Op_MaxHF:
2333 case Op_SqrtHF:
2334 // Half-precision floating point scalar operations require FEAT_FP16
2335 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2336 // features are supported.
2337 if (!is_feat_fp16_supported()) {
2338 return false;
2339 }
2340 break;
2341 }
2342
2343 return true; // Per default match rules are supported.
2344 }
2345
2346 const RegMask* Matcher::predicate_reg_mask(void) {
2347 return &_PR_REG_mask;
2348 }
2349
2350 bool Matcher::supports_vector_calling_convention(void) {
2351 return EnableVectorSupport;
2352 }
2353
2354 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2355 assert(EnableVectorSupport, "sanity");
2356 int lo = V0_num;
2357 int hi = V0_H_num;
2358 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2359 hi = V0_K_num;
2360 }
2361 return OptoRegPair(hi, lo);
2362 }
2363
2364 // Is this branch offset short enough that a short branch can be used?
2365 //
2366 // NOTE: If the platform does not provide any short branch variants, then
2367 // this method should return false for offset 0.
2368 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2369 // The passed offset is relative to address of the branch.
2370
2371 return (-32768 <= offset && offset < 32768);
2372 }
2373
2374 // Vector width in bytes.
2375 int Matcher::vector_width_in_bytes(BasicType bt) {
2376 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2377 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2378 // Minimum 2 values in vector
2379 if (size < 2*type2aelembytes(bt)) size = 0;
2380 // But never < 4
2381 if (size < 4) size = 0;
2382 return size;
2383 }
2384
2385 // Limits on vector size (number of elements) loaded into vector.
2386 int Matcher::max_vector_size(const BasicType bt) {
2387 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2388 }
2389
2390 int Matcher::min_vector_size(const BasicType bt) {
2391 // Usually, the shortest vector length supported by AArch64 ISA and
2392 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2393 // vectors in a few special cases.
2394 int size;
2395 switch(bt) {
2396 case T_BOOLEAN:
2397 // Load/store a vector mask with only 2 elements for vector types
2398 // such as "2I/2F/2L/2D".
2399 size = 2;
2400 break;
2401 case T_BYTE:
2402 // Generate a "4B" vector, to support vector cast between "8B/16B"
2403 // and "4S/4I/4L/4F/4D".
2404 size = 4;
2405 break;
2406 case T_SHORT:
2407 // Generate a "2S" vector, to support vector cast between "4S/8S"
2408 // and "2I/2L/2F/2D".
2409 size = 2;
2410 break;
2411 default:
2412 // Limit the min vector length to 64-bit.
2413 size = 8 / type2aelembytes(bt);
2414 // The number of elements in a vector should be at least 2.
2415 size = MAX2(size, 2);
2416 }
2417
2418 int max_size = max_vector_size(bt);
2419 return MIN2(size, max_size);
2420 }
2421
2422 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2423 return Matcher::max_vector_size(bt);
2424 }
2425
2426 // Actual max scalable vector register length.
2427 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2428 return Matcher::max_vector_size(bt);
2429 }
2430
2431 // Vector ideal reg.
2432 uint Matcher::vector_ideal_reg(int len) {
2433 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2434 return Op_VecA;
2435 }
2436 switch(len) {
2437 // For 16-bit/32-bit mask vector, reuse VecD.
2438 case 2:
2439 case 4:
2440 case 8: return Op_VecD;
2441 case 16: return Op_VecX;
2442 }
2443 ShouldNotReachHere();
2444 return 0;
2445 }
2446
2447 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2448 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2449 switch (ideal_reg) {
2450 case Op_VecA: return new vecAOper();
2451 case Op_VecD: return new vecDOper();
2452 case Op_VecX: return new vecXOper();
2453 }
2454 ShouldNotReachHere();
2455 return nullptr;
2456 }
2457
2458 bool Matcher::is_reg2reg_move(MachNode* m) {
2459 return false;
2460 }
2461
2462 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2463 return false;
2464 }
2465
2466 bool Matcher::is_generic_vector(MachOper* opnd) {
2467 return opnd->opcode() == VREG;
2468 }
2469
2470 // Return whether or not this register is ever used as an argument.
2471 // This function is used on startup to build the trampoline stubs in
2472 // generateOptoStub. Registers not mentioned will be killed by the VM
2473 // call in the trampoline, and arguments in those registers not be
2474 // available to the callee.
2475 bool Matcher::can_be_java_arg(int reg)
2476 {
2477 return
2478 reg == R0_num || reg == R0_H_num ||
2479 reg == R1_num || reg == R1_H_num ||
2480 reg == R2_num || reg == R2_H_num ||
2481 reg == R3_num || reg == R3_H_num ||
2482 reg == R4_num || reg == R4_H_num ||
2483 reg == R5_num || reg == R5_H_num ||
2484 reg == R6_num || reg == R6_H_num ||
2485 reg == R7_num || reg == R7_H_num ||
2486 reg == V0_num || reg == V0_H_num ||
2487 reg == V1_num || reg == V1_H_num ||
2488 reg == V2_num || reg == V2_H_num ||
2489 reg == V3_num || reg == V3_H_num ||
2490 reg == V4_num || reg == V4_H_num ||
2491 reg == V5_num || reg == V5_H_num ||
2492 reg == V6_num || reg == V6_H_num ||
2493 reg == V7_num || reg == V7_H_num;
2494 }
2495
2496 bool Matcher::is_spillable_arg(int reg)
2497 {
2498 return can_be_java_arg(reg);
2499 }
2500
2501 uint Matcher::int_pressure_limit()
2502 {
2503 // JDK-8183543: When taking the number of available registers as int
2504 // register pressure threshold, the jtreg test:
2505 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2506 // failed due to C2 compilation failure with
2507 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2508 //
2509 // A derived pointer is live at CallNode and then is flagged by RA
2510 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2511 // derived pointers and lastly fail to spill after reaching maximum
2512 // number of iterations. Lowering the default pressure threshold to
2513 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2514 // a high register pressure area of the code so that split_DEF can
2515 // generate DefinitionSpillCopy for the derived pointer.
2516 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2517 if (!PreserveFramePointer) {
2518 // When PreserveFramePointer is off, frame pointer is allocatable,
2519 // but different from other SOC registers, it is excluded from
2520 // fatproj's mask because its save type is No-Save. Decrease 1 to
2521 // ensure high pressure at fatproj when PreserveFramePointer is off.
2522 // See check_pressure_at_fatproj().
2523 default_int_pressure_threshold--;
2524 }
2525 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2526 }
2527
2528 uint Matcher::float_pressure_limit()
2529 {
2530 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2531 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2532 }
2533
2534 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2535 return false;
2536 }
2537
2538 const RegMask& Matcher::divI_proj_mask() {
2539 ShouldNotReachHere();
2540 return RegMask::EMPTY;
2541 }
2542
2543 // Register for MODI projection of divmodI.
2544 const RegMask& Matcher::modI_proj_mask() {
2545 ShouldNotReachHere();
2546 return RegMask::EMPTY;
2547 }
2548
2549 // Register for DIVL projection of divmodL.
2550 const RegMask& Matcher::divL_proj_mask() {
2551 ShouldNotReachHere();
2552 return RegMask::EMPTY;
2553 }
2554
2555 // Register for MODL projection of divmodL.
2556 const RegMask& Matcher::modL_proj_mask() {
2557 ShouldNotReachHere();
2558 return RegMask::EMPTY;
2559 }
2560
2561 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2562 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2563 Node* u = addp->fast_out(i);
2564 if (u->is_LoadStore()) {
2565 // On AArch64, LoadStoreNodes (i.e. compare and swap
2566 // instructions) only take register indirect as an operand, so
2567 // any attempt to use an AddPNode as an input to a LoadStoreNode
2568 // must fail.
2569 return false;
2570 }
2571 if (u->is_Mem()) {
2572 int opsize = u->as_Mem()->memory_size();
2573 assert(opsize > 0, "unexpected memory operand size");
2574 if (u->as_Mem()->memory_size() != (1<<shift)) {
2575 return false;
2576 }
2577 }
2578 }
2579 return true;
2580 }
2581
2582 // Convert BoolTest condition to Assembler condition.
2583 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2584 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2585 Assembler::Condition result;
2586 switch(cond) {
2587 case BoolTest::eq:
2588 result = Assembler::EQ; break;
2589 case BoolTest::ne:
2590 result = Assembler::NE; break;
2591 case BoolTest::le:
2592 result = Assembler::LE; break;
2593 case BoolTest::ge:
2594 result = Assembler::GE; break;
2595 case BoolTest::lt:
2596 result = Assembler::LT; break;
2597 case BoolTest::gt:
2598 result = Assembler::GT; break;
2599 case BoolTest::ule:
2600 result = Assembler::LS; break;
2601 case BoolTest::uge:
2602 result = Assembler::HS; break;
2603 case BoolTest::ult:
2604 result = Assembler::LO; break;
2605 case BoolTest::ugt:
2606 result = Assembler::HI; break;
2607 case BoolTest::overflow:
2608 result = Assembler::VS; break;
2609 case BoolTest::no_overflow:
2610 result = Assembler::VC; break;
2611 default:
2612 ShouldNotReachHere();
2613 return Assembler::Condition(-1);
2614 }
2615
2616 // Check conversion
2617 if (cond & BoolTest::unsigned_compare) {
2618 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2619 } else {
2620 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2621 }
2622
2623 return result;
2624 }
2625
2626 // Binary src (Replicate con)
2627 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2628 if (n == nullptr || m == nullptr) {
2629 return false;
2630 }
2631
2632 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2633 return false;
2634 }
2635
2636 Node* imm_node = m->in(1);
2637 if (!imm_node->is_Con()) {
2638 return false;
2639 }
2640
2641 const Type* t = imm_node->bottom_type();
2642 if (!(t->isa_int() || t->isa_long())) {
2643 return false;
2644 }
2645
2646 switch (n->Opcode()) {
2647 case Op_AndV:
2648 case Op_OrV:
2649 case Op_XorV: {
2650 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2651 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2652 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2653 }
2654 case Op_AddVB:
2655 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2656 case Op_AddVS:
2657 case Op_AddVI:
2658 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2659 case Op_AddVL:
2660 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2661 default:
2662 return false;
2663 }
2664 }
2665
2666 // (XorV src (Replicate m1))
2667 // (XorVMask src (MaskAll m1))
2668 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2669 if (n != nullptr && m != nullptr) {
2670 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2671 VectorNode::is_all_ones_vector(m);
2672 }
2673 return false;
2674 }
2675
2676 // Should the matcher clone input 'm' of node 'n'?
2677 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2678 if (is_vshift_con_pattern(n, m) ||
2679 is_vector_bitwise_not_pattern(n, m) ||
2680 is_valid_sve_arith_imm_pattern(n, m) ||
2681 is_encode_and_store_pattern(n, m)) {
2682 mstack.push(m, Visit);
2683 return true;
2684 }
2685 return false;
2686 }
2687
2688 // Should the Matcher clone shifts on addressing modes, expecting them
2689 // to be subsumed into complex addressing expressions or compute them
2690 // into registers?
2691 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2692
2693 // Loads and stores with indirect memory input (e.g., volatile loads and
2694 // stores) do not subsume the input into complex addressing expressions. If
2695 // the addressing expression is input to at least one such load or store, do
2696 // not clone the addressing expression. Query needs_acquiring_load and
2697 // needs_releasing_store as a proxy for indirect memory input, as it is not
2698 // possible to directly query for indirect memory input at this stage.
2699 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2700 Node* n = m->fast_out(i);
2701 if (n->is_Load() && needs_acquiring_load(n)) {
2702 return false;
2703 }
2704 if (n->is_Store() && needs_releasing_store(n)) {
2705 return false;
2706 }
2707 }
2708
2709 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2710 return true;
2711 }
2712
2713 Node *off = m->in(AddPNode::Offset);
2714 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2715 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2716 // Are there other uses besides address expressions?
2717 !is_visited(off)) {
2718 address_visited.set(off->_idx); // Flag as address_visited
2719 mstack.push(off->in(2), Visit);
2720 Node *conv = off->in(1);
2721 if (conv->Opcode() == Op_ConvI2L &&
2722 // Are there other uses besides address expressions?
2723 !is_visited(conv)) {
2724 address_visited.set(conv->_idx); // Flag as address_visited
2725 mstack.push(conv->in(1), Pre_Visit);
2726 } else {
2727 mstack.push(conv, Pre_Visit);
2728 }
2729 address_visited.test_set(m->_idx); // Flag as address_visited
2730 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2731 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2732 return true;
2733 } else if (off->Opcode() == Op_ConvI2L &&
2734 // Are there other uses besides address expressions?
2735 !is_visited(off)) {
2736 address_visited.test_set(m->_idx); // Flag as address_visited
2737 address_visited.set(off->_idx); // Flag as address_visited
2738 mstack.push(off->in(1), Pre_Visit);
2739 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2740 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2741 return true;
2742 }
2743 return false;
2744 }
2745
2746 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2747 { \
2748 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2749 guarantee(DISP == 0, "mode not permitted for volatile"); \
2750 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2751 __ INSN(REG, as_Register(BASE)); \
2752 }
2753
2754
2755 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2756 {
2757 Address::extend scale;
2758
2759 // Hooboy, this is fugly. We need a way to communicate to the
2760 // encoder that the index needs to be sign extended, so we have to
2761 // enumerate all the cases.
2762 switch (opcode) {
2763 case INDINDEXSCALEDI2L:
2764 case INDINDEXSCALEDI2LN:
2765 case INDINDEXI2L:
2766 case INDINDEXI2LN:
2767 scale = Address::sxtw(size);
2768 break;
2769 default:
2770 scale = Address::lsl(size);
2771 }
2772
2773 if (index == -1) {
2774 return Address(base, disp);
2775 } else {
2776 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2777 return Address(base, as_Register(index), scale);
2778 }
2779 }
2780
2781
2782 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2783 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2784 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2785 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2786 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2787
2788 // Used for all non-volatile memory accesses. The use of
2789 // $mem->opcode() to discover whether this pattern uses sign-extended
2790 // offsets is something of a kludge.
2791 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2792 Register reg, int opcode,
2793 Register base, int index, int scale, int disp,
2794 int size_in_memory)
2795 {
2796 Address addr = mem2address(opcode, base, index, scale, disp);
2797 if (addr.getMode() == Address::base_plus_offset) {
2798 /* Fix up any out-of-range offsets. */
2799 assert_different_registers(rscratch1, base);
2800 assert_different_registers(rscratch1, reg);
2801 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2802 }
2803 (masm->*insn)(reg, addr);
2804 }
2805
2806 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2807 FloatRegister reg, int opcode,
2808 Register base, int index, int size, int disp,
2809 int size_in_memory)
2810 {
2811 Address::extend scale;
2812
2813 switch (opcode) {
2814 case INDINDEXSCALEDI2L:
2815 case INDINDEXSCALEDI2LN:
2816 scale = Address::sxtw(size);
2817 break;
2818 default:
2819 scale = Address::lsl(size);
2820 }
2821
2822 if (index == -1) {
2823 // Fix up any out-of-range offsets.
2824 assert_different_registers(rscratch1, base);
2825 Address addr = Address(base, disp);
2826 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2827 (masm->*insn)(reg, addr);
2828 } else {
2829 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2830 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2831 }
2832 }
2833
2834 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2835 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2836 int opcode, Register base, int index, int size, int disp)
2837 {
2838 if (index == -1) {
2839 (masm->*insn)(reg, T, Address(base, disp));
2840 } else {
2841 assert(disp == 0, "unsupported address mode");
2842 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2843 }
2844 }
2845
2846 %}
2847
2848
2849
2850 //----------ENCODING BLOCK-----------------------------------------------------
2851 // This block specifies the encoding classes used by the compiler to
2852 // output byte streams. Encoding classes are parameterized macros
2853 // used by Machine Instruction Nodes in order to generate the bit
2854 // encoding of the instruction. Operands specify their base encoding
2855 // interface with the interface keyword. There are currently
2856 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2857 // COND_INTER. REG_INTER causes an operand to generate a function
2858 // which returns its register number when queried. CONST_INTER causes
2859 // an operand to generate a function which returns the value of the
2860 // constant when queried. MEMORY_INTER causes an operand to generate
2861 // four functions which return the Base Register, the Index Register,
2862 // the Scale Value, and the Offset Value of the operand when queried.
2863 // COND_INTER causes an operand to generate six functions which return
2864 // the encoding code (ie - encoding bits for the instruction)
2865 // associated with each basic boolean condition for a conditional
2866 // instruction.
2867 //
2868 // Instructions specify two basic values for encoding. Again, a
2869 // function is available to check if the constant displacement is an
2870 // oop. They use the ins_encode keyword to specify their encoding
2871 // classes (which must be a sequence of enc_class names, and their
2872 // parameters, specified in the encoding block), and they use the
2873 // opcode keyword to specify, in order, their primary, secondary, and
2874 // tertiary opcode. Only the opcode sections which a particular
2875 // instruction needs for encoding need to be specified.
2876 encode %{
2877 // Build emit functions for each basic byte or larger field in the
2878 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2879 // from C++ code in the enc_class source block. Emit functions will
2880 // live in the main source block for now. In future, we can
2881 // generalize this by adding a syntax that specifies the sizes of
2882 // fields in an order, so that the adlc can build the emit functions
2883 // automagically
2884
2885 // catch all for unimplemented encodings
2886 enc_class enc_unimplemented %{
2887 __ unimplemented("C2 catch all");
2888 %}
2889
2890 // BEGIN Non-volatile memory access
2891
2892 // This encoding class is generated automatically from ad_encode.m4.
2893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2894 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2895 Register dst_reg = as_Register($dst$$reg);
2896 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2897 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2898 %}
2899
2900 // This encoding class is generated automatically from ad_encode.m4.
2901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2902 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2903 Register dst_reg = as_Register($dst$$reg);
2904 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2905 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2906 %}
2907
2908 // This encoding class is generated automatically from ad_encode.m4.
2909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2910 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2911 Register dst_reg = as_Register($dst$$reg);
2912 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2913 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2914 %}
2915
2916 // This encoding class is generated automatically from ad_encode.m4.
2917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2918 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2919 Register dst_reg = as_Register($dst$$reg);
2920 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2921 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2922 %}
2923
2924 // This encoding class is generated automatically from ad_encode.m4.
2925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2926 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2927 Register dst_reg = as_Register($dst$$reg);
2928 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2929 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2930 %}
2931
2932 // This encoding class is generated automatically from ad_encode.m4.
2933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2934 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2935 Register dst_reg = as_Register($dst$$reg);
2936 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2937 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2938 %}
2939
2940 // This encoding class is generated automatically from ad_encode.m4.
2941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2942 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2943 Register dst_reg = as_Register($dst$$reg);
2944 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2945 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2946 %}
2947
2948 // This encoding class is generated automatically from ad_encode.m4.
2949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2950 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2951 Register dst_reg = as_Register($dst$$reg);
2952 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2953 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2954 %}
2955
2956 // This encoding class is generated automatically from ad_encode.m4.
2957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2958 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2959 Register dst_reg = as_Register($dst$$reg);
2960 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2961 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2962 %}
2963
2964 // This encoding class is generated automatically from ad_encode.m4.
2965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2966 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2967 Register dst_reg = as_Register($dst$$reg);
2968 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2969 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2970 %}
2971
2972 // This encoding class is generated automatically from ad_encode.m4.
2973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2974 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2975 Register dst_reg = as_Register($dst$$reg);
2976 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2977 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2978 %}
2979
2980 // This encoding class is generated automatically from ad_encode.m4.
2981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2982 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2983 Register dst_reg = as_Register($dst$$reg);
2984 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2985 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2986 %}
2987
2988 // This encoding class is generated automatically from ad_encode.m4.
2989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2990 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2991 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2992 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2993 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2994 %}
2995
2996 // This encoding class is generated automatically from ad_encode.m4.
2997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2998 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2999 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3000 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3001 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3002 %}
3003
3004 // This encoding class is generated automatically from ad_encode.m4.
3005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3006 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3007 Register src_reg = as_Register($src$$reg);
3008 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3009 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3010 %}
3011
3012 // This encoding class is generated automatically from ad_encode.m4.
3013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3014 enc_class aarch64_enc_strb0(memory1 mem) %{
3015 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3022 Register src_reg = as_Register($src$$reg);
3023 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strh0(memory2 mem) %{
3030 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3037 Register src_reg = as_Register($src$$reg);
3038 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_strw0(memory4 mem) %{
3045 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3046 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3047 %}
3048
3049 // This encoding class is generated automatically from ad_encode.m4.
3050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3051 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3052 Register src_reg = as_Register($src$$reg);
3053 // we sometimes get asked to store the stack pointer into the
3054 // current thread -- we cannot do that directly on AArch64
3055 if (src_reg == r31_sp) {
3056 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3057 __ mov(rscratch2, sp);
3058 src_reg = rscratch2;
3059 }
3060 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_str0(memory8 mem) %{
3067 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3068 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3069 %}
3070
3071 // This encoding class is generated automatically from ad_encode.m4.
3072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3073 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3074 FloatRegister src_reg = as_FloatRegister($src$$reg);
3075 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3076 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3077 %}
3078
3079 // This encoding class is generated automatically from ad_encode.m4.
3080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3081 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3082 FloatRegister src_reg = as_FloatRegister($src$$reg);
3083 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3084 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3085 %}
3086
3087 // This encoding class is generated automatically from ad_encode.m4.
3088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3089 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3090 __ membar(Assembler::StoreStore);
3091 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3092 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3093 %}
3094
3095 // END Non-volatile memory access
3096
3097 // Vector loads and stores
3098 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3099 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3100 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3101 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3102 %}
3103
3104 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3105 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3106 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3107 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3108 %}
3109
3110 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3111 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3112 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3113 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3114 %}
3115
3116 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3117 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3118 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3119 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3120 %}
3121
3122 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3123 FloatRegister src_reg = as_FloatRegister($src$$reg);
3124 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3125 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3126 %}
3127
3128 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3129 FloatRegister src_reg = as_FloatRegister($src$$reg);
3130 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3131 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3132 %}
3133
3134 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3135 FloatRegister src_reg = as_FloatRegister($src$$reg);
3136 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3137 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3138 %}
3139
3140 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3141 FloatRegister src_reg = as_FloatRegister($src$$reg);
3142 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3143 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3144 %}
3145
3146 // volatile loads and stores
3147
3148 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3149 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3150 rscratch1, stlrb);
3151 %}
3152
3153 enc_class aarch64_enc_stlrb0(memory mem) %{
3154 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, stlrb);
3156 %}
3157
3158 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3159 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, stlrh);
3161 %}
3162
3163 enc_class aarch64_enc_stlrh0(memory mem) %{
3164 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, stlrh);
3166 %}
3167
3168 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3169 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3170 rscratch1, stlrw);
3171 %}
3172
3173 enc_class aarch64_enc_stlrw0(memory mem) %{
3174 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, stlrw);
3176 %}
3177
3178 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3179 Register dst_reg = as_Register($dst$$reg);
3180 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarb);
3182 __ sxtbw(dst_reg, dst_reg);
3183 %}
3184
3185 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3186 Register dst_reg = as_Register($dst$$reg);
3187 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarb);
3189 __ sxtb(dst_reg, dst_reg);
3190 %}
3191
3192 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3193 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarb);
3195 %}
3196
3197 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3198 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, ldarb);
3200 %}
3201
3202 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3203 Register dst_reg = as_Register($dst$$reg);
3204 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarh);
3206 __ sxthw(dst_reg, dst_reg);
3207 %}
3208
3209 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3210 Register dst_reg = as_Register($dst$$reg);
3211 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, ldarh);
3213 __ sxth(dst_reg, dst_reg);
3214 %}
3215
3216 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3217 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, ldarh);
3219 %}
3220
3221 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3222 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3223 rscratch1, ldarh);
3224 %}
3225
3226 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3227 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3228 rscratch1, ldarw);
3229 %}
3230
3231 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3232 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3233 rscratch1, ldarw);
3234 %}
3235
3236 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3237 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, ldar);
3239 %}
3240
3241 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3242 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3243 rscratch1, ldarw);
3244 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3245 %}
3246
3247 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3248 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3249 rscratch1, ldar);
3250 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3251 %}
3252
3253 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3254 Register src_reg = as_Register($src$$reg);
3255 // we sometimes get asked to store the stack pointer into the
3256 // current thread -- we cannot do that directly on AArch64
3257 if (src_reg == r31_sp) {
3258 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3259 __ mov(rscratch2, sp);
3260 src_reg = rscratch2;
3261 }
3262 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, stlr);
3264 %}
3265
3266 enc_class aarch64_enc_stlr0(memory mem) %{
3267 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3268 rscratch1, stlr);
3269 %}
3270
3271 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3272 {
3273 FloatRegister src_reg = as_FloatRegister($src$$reg);
3274 __ fmovs(rscratch2, src_reg);
3275 }
3276 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, stlrw);
3278 %}
3279
3280 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3281 {
3282 FloatRegister src_reg = as_FloatRegister($src$$reg);
3283 __ fmovd(rscratch2, src_reg);
3284 }
3285 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3286 rscratch1, stlr);
3287 %}
3288
3289 // synchronized read/update encodings
3290
3291 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3292 Register dst_reg = as_Register($dst$$reg);
3293 Register base = as_Register($mem$$base);
3294 int index = $mem$$index;
3295 int scale = $mem$$scale;
3296 int disp = $mem$$disp;
3297 if (index == -1) {
3298 if (disp != 0) {
3299 __ lea(rscratch1, Address(base, disp));
3300 __ ldaxr(dst_reg, rscratch1);
3301 } else {
3302 // TODO
3303 // should we ever get anything other than this case?
3304 __ ldaxr(dst_reg, base);
3305 }
3306 } else {
3307 Register index_reg = as_Register(index);
3308 if (disp == 0) {
3309 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3310 __ ldaxr(dst_reg, rscratch1);
3311 } else {
3312 __ lea(rscratch1, Address(base, disp));
3313 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3314 __ ldaxr(dst_reg, rscratch1);
3315 }
3316 }
3317 %}
3318
3319 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3320 Register src_reg = as_Register($src$$reg);
3321 Register base = as_Register($mem$$base);
3322 int index = $mem$$index;
3323 int scale = $mem$$scale;
3324 int disp = $mem$$disp;
3325 if (index == -1) {
3326 if (disp != 0) {
3327 __ lea(rscratch2, Address(base, disp));
3328 __ stlxr(rscratch1, src_reg, rscratch2);
3329 } else {
3330 // TODO
3331 // should we ever get anything other than this case?
3332 __ stlxr(rscratch1, src_reg, base);
3333 }
3334 } else {
3335 Register index_reg = as_Register(index);
3336 if (disp == 0) {
3337 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3338 __ stlxr(rscratch1, src_reg, rscratch2);
3339 } else {
3340 __ lea(rscratch2, Address(base, disp));
3341 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3342 __ stlxr(rscratch1, src_reg, rscratch2);
3343 }
3344 }
3345 __ cmpw(rscratch1, zr);
3346 %}
3347
3348 // prefetch encodings
3349
3350 enc_class aarch64_enc_prefetchw(memory mem) %{
3351 Register base = as_Register($mem$$base);
3352 int index = $mem$$index;
3353 int scale = $mem$$scale;
3354 int disp = $mem$$disp;
3355 if (index == -1) {
3356 // Fix up any out-of-range offsets.
3357 assert_different_registers(rscratch1, base);
3358 Address addr = Address(base, disp);
3359 addr = __ legitimize_address(addr, 8, rscratch1);
3360 __ prfm(addr, PSTL1KEEP);
3361 } else {
3362 Register index_reg = as_Register(index);
3363 if (disp == 0) {
3364 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3365 } else {
3366 __ lea(rscratch1, Address(base, disp));
3367 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3368 }
3369 }
3370 %}
3371
3372 // mov encodings
3373
3374 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3375 uint32_t con = (uint32_t)$src$$constant;
3376 Register dst_reg = as_Register($dst$$reg);
3377 if (con == 0) {
3378 __ movw(dst_reg, zr);
3379 } else {
3380 __ movw(dst_reg, con);
3381 }
3382 %}
3383
3384 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3385 Register dst_reg = as_Register($dst$$reg);
3386 uint64_t con = (uint64_t)$src$$constant;
3387 if (con == 0) {
3388 __ mov(dst_reg, zr);
3389 } else {
3390 __ mov(dst_reg, con);
3391 }
3392 %}
3393
3394 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3395 Register dst_reg = as_Register($dst$$reg);
3396 address con = (address)$src$$constant;
3397 if (con == nullptr || con == (address)1) {
3398 ShouldNotReachHere();
3399 } else {
3400 relocInfo::relocType rtype = $src->constant_reloc();
3401 if (rtype == relocInfo::oop_type) {
3402 __ movoop(dst_reg, (jobject)con);
3403 } else if (rtype == relocInfo::metadata_type) {
3404 __ mov_metadata(dst_reg, (Metadata*)con);
3405 } else {
3406 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3407 // load fake address constants using a normal move
3408 if (! __ is_valid_AArch64_address(con) ||
3409 con < (address)(uintptr_t)os::vm_page_size()) {
3410 __ mov(dst_reg, con);
3411 } else {
3412 // no reloc so just use adrp and add
3413 uint64_t offset;
3414 __ adrp(dst_reg, con, offset);
3415 __ add(dst_reg, dst_reg, offset);
3416 }
3417 }
3418 }
3419 %}
3420
3421 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3422 Register dst_reg = as_Register($dst$$reg);
3423 __ mov(dst_reg, zr);
3424 %}
3425
3426 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, (uint64_t)1);
3429 %}
3430
3431 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3432 Register dst_reg = as_Register($dst$$reg);
3433 address con = (address)$src$$constant;
3434 if (con == nullptr) {
3435 ShouldNotReachHere();
3436 } else {
3437 relocInfo::relocType rtype = $src->constant_reloc();
3438 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3439 __ set_narrow_oop(dst_reg, (jobject)con);
3440 }
3441 %}
3442
3443 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3444 Register dst_reg = as_Register($dst$$reg);
3445 __ mov(dst_reg, zr);
3446 %}
3447
3448 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3449 Register dst_reg = as_Register($dst$$reg);
3450 address con = (address)$src$$constant;
3451 if (con == nullptr) {
3452 ShouldNotReachHere();
3453 } else {
3454 relocInfo::relocType rtype = $src->constant_reloc();
3455 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3456 __ set_narrow_klass(dst_reg, (Klass *)con);
3457 }
3458 %}
3459
3460 // arithmetic encodings
3461
3462 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3463 Register dst_reg = as_Register($dst$$reg);
3464 Register src_reg = as_Register($src1$$reg);
3465 int32_t con = (int32_t)$src2$$constant;
3466 // add has primary == 0, subtract has primary == 1
3467 if ($primary) { con = -con; }
3468 if (con < 0) {
3469 __ subw(dst_reg, src_reg, -con);
3470 } else {
3471 __ addw(dst_reg, src_reg, con);
3472 }
3473 %}
3474
3475 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3476 Register dst_reg = as_Register($dst$$reg);
3477 Register src_reg = as_Register($src1$$reg);
3478 int32_t con = (int32_t)$src2$$constant;
3479 // add has primary == 0, subtract has primary == 1
3480 if ($primary) { con = -con; }
3481 if (con < 0) {
3482 __ sub(dst_reg, src_reg, -con);
3483 } else {
3484 __ add(dst_reg, src_reg, con);
3485 }
3486 %}
3487
3488 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3489 Register dst_reg = as_Register($dst$$reg);
3490 Register src1_reg = as_Register($src1$$reg);
3491 Register src2_reg = as_Register($src2$$reg);
3492 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3493 %}
3494
3495 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3496 Register dst_reg = as_Register($dst$$reg);
3497 Register src1_reg = as_Register($src1$$reg);
3498 Register src2_reg = as_Register($src2$$reg);
3499 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3500 %}
3501
3502 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3503 Register dst_reg = as_Register($dst$$reg);
3504 Register src1_reg = as_Register($src1$$reg);
3505 Register src2_reg = as_Register($src2$$reg);
3506 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3507 %}
3508
3509 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3510 Register dst_reg = as_Register($dst$$reg);
3511 Register src1_reg = as_Register($src1$$reg);
3512 Register src2_reg = as_Register($src2$$reg);
3513 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3514 %}
3515
3516 // compare instruction encodings
3517
3518 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3519 Register reg1 = as_Register($src1$$reg);
3520 Register reg2 = as_Register($src2$$reg);
3521 __ cmpw(reg1, reg2);
3522 %}
3523
3524 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3525 Register reg = as_Register($src1$$reg);
3526 int32_t val = $src2$$constant;
3527 if (val >= 0) {
3528 __ subsw(zr, reg, val);
3529 } else {
3530 __ addsw(zr, reg, -val);
3531 }
3532 %}
3533
3534 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3535 Register reg1 = as_Register($src1$$reg);
3536 uint32_t val = (uint32_t)$src2$$constant;
3537 __ movw(rscratch1, val);
3538 __ cmpw(reg1, rscratch1);
3539 %}
3540
3541 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3542 Register reg1 = as_Register($src1$$reg);
3543 Register reg2 = as_Register($src2$$reg);
3544 __ cmp(reg1, reg2);
3545 %}
3546
3547 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3548 Register reg = as_Register($src1$$reg);
3549 int64_t val = $src2$$constant;
3550 if (val >= 0) {
3551 __ subs(zr, reg, val);
3552 } else if (val != -val) {
3553 __ adds(zr, reg, -val);
3554 } else {
3555 // aargh, Long.MIN_VALUE is a special case
3556 __ orr(rscratch1, zr, (uint64_t)val);
3557 __ subs(zr, reg, rscratch1);
3558 }
3559 %}
3560
3561 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3562 Register reg1 = as_Register($src1$$reg);
3563 uint64_t val = (uint64_t)$src2$$constant;
3564 __ mov(rscratch1, val);
3565 __ cmp(reg1, rscratch1);
3566 %}
3567
3568 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3569 Register reg1 = as_Register($src1$$reg);
3570 Register reg2 = as_Register($src2$$reg);
3571 __ cmp(reg1, reg2);
3572 %}
3573
3574 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3575 Register reg1 = as_Register($src1$$reg);
3576 Register reg2 = as_Register($src2$$reg);
3577 __ cmpw(reg1, reg2);
3578 %}
3579
3580 enc_class aarch64_enc_testp(iRegP src) %{
3581 Register reg = as_Register($src$$reg);
3582 __ cmp(reg, zr);
3583 %}
3584
3585 enc_class aarch64_enc_testn(iRegN src) %{
3586 Register reg = as_Register($src$$reg);
3587 __ cmpw(reg, zr);
3588 %}
3589
3590 enc_class aarch64_enc_b(label lbl) %{
3591 Label *L = $lbl$$label;
3592 __ b(*L);
3593 %}
3594
3595 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3596 Label *L = $lbl$$label;
3597 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3598 %}
3599
3600 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3601 Label *L = $lbl$$label;
3602 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3603 %}
3604
3605 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3606 %{
3607 Register sub_reg = as_Register($sub$$reg);
3608 Register super_reg = as_Register($super$$reg);
3609 Register temp_reg = as_Register($temp$$reg);
3610 Register result_reg = as_Register($result$$reg);
3611
3612 Label miss;
3613 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3614 nullptr, &miss,
3615 /*set_cond_codes:*/ true);
3616 if ($primary) {
3617 __ mov(result_reg, zr);
3618 }
3619 __ bind(miss);
3620 %}
3621
3622 enc_class aarch64_enc_java_static_call(method meth) %{
3623 address addr = (address)$meth$$method;
3624 address call;
3625 if (!_method) {
3626 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3627 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3628 if (call == nullptr) {
3629 ciEnv::current()->record_failure("CodeCache is full");
3630 return;
3631 }
3632 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3633 // The NOP here is purely to ensure that eliding a call to
3634 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3635 __ nop();
3636 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3637 } else {
3638 int method_index = resolved_method_index(masm);
3639 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3640 : static_call_Relocation::spec(method_index);
3641 call = __ trampoline_call(Address(addr, rspec));
3642 if (call == nullptr) {
3643 ciEnv::current()->record_failure("CodeCache is full");
3644 return;
3645 }
3646 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3647 // Calls of the same statically bound method can share
3648 // a stub to the interpreter.
3649 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3650 } else {
3651 // Emit stub for static call
3652 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3653 if (stub == nullptr) {
3654 ciEnv::current()->record_failure("CodeCache is full");
3655 return;
3656 }
3657 }
3658 }
3659
3660 __ post_call_nop();
3661
3662 // Only non uncommon_trap calls need to reinitialize ptrue.
3663 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3664 __ reinitialize_ptrue();
3665 }
3666 %}
3667
3668 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3669 int method_index = resolved_method_index(masm);
3670 address call = __ ic_call((address)$meth$$method, method_index);
3671 if (call == nullptr) {
3672 ciEnv::current()->record_failure("CodeCache is full");
3673 return;
3674 }
3675 __ post_call_nop();
3676 if (Compile::current()->max_vector_size() > 0) {
3677 __ reinitialize_ptrue();
3678 }
3679 %}
3680
3681 enc_class aarch64_enc_call_epilog() %{
3682 if (VerifyStackAtCalls) {
3683 // Check that stack depth is unchanged: find majik cookie on stack
3684 __ call_Unimplemented();
3685 }
3686 %}
3687
3688 enc_class aarch64_enc_java_to_runtime(method meth) %{
3689 // some calls to generated routines (arraycopy code) are scheduled
3690 // by C2 as runtime calls. if so we can call them using a br (they
3691 // will be in a reachable segment) otherwise we have to use a blr
3692 // which loads the absolute address into a register.
3693 address entry = (address)$meth$$method;
3694 CodeBlob *cb = CodeCache::find_blob(entry);
3695 if (cb) {
3696 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3697 if (call == nullptr) {
3698 ciEnv::current()->record_failure("CodeCache is full");
3699 return;
3700 }
3701 __ post_call_nop();
3702 } else {
3703 Label retaddr;
3704 // Make the anchor frame walkable
3705 __ adr(rscratch2, retaddr);
3706 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3707 __ lea(rscratch1, RuntimeAddress(entry));
3708 __ blr(rscratch1);
3709 __ bind(retaddr);
3710 __ post_call_nop();
3711 }
3712 if (Compile::current()->max_vector_size() > 0) {
3713 __ reinitialize_ptrue();
3714 }
3715 %}
3716
3717 enc_class aarch64_enc_rethrow() %{
3718 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3719 %}
3720
3721 enc_class aarch64_enc_ret() %{
3722 #ifdef ASSERT
3723 if (Compile::current()->max_vector_size() > 0) {
3724 __ verify_ptrue();
3725 }
3726 #endif
3727 __ ret(lr);
3728 %}
3729
3730 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3731 Register target_reg = as_Register($jump_target$$reg);
3732 __ br(target_reg);
3733 %}
3734
3735 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3736 Register target_reg = as_Register($jump_target$$reg);
3737 // exception oop should be in r0
3738 // ret addr has been popped into lr
3739 // callee expects it in r3
3740 __ mov(r3, lr);
3741 __ br(target_reg);
3742 %}
3743
3744 %}
3745
3746 //----------FRAME--------------------------------------------------------------
3747 // Definition of frame structure and management information.
3748 //
3749 // S T A C K L A Y O U T Allocators stack-slot number
3750 // | (to get allocators register number
3751 // G Owned by | | v add OptoReg::stack0())
3752 // r CALLER | |
3753 // o | +--------+ pad to even-align allocators stack-slot
3754 // w V | pad0 | numbers; owned by CALLER
3755 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3756 // h ^ | in | 5
3757 // | | args | 4 Holes in incoming args owned by SELF
3758 // | | | | 3
3759 // | | +--------+
3760 // V | | old out| Empty on Intel, window on Sparc
3761 // | old |preserve| Must be even aligned.
3762 // | SP-+--------+----> Matcher::_old_SP, even aligned
3763 // | | in | 3 area for Intel ret address
3764 // Owned by |preserve| Empty on Sparc.
3765 // SELF +--------+
3766 // | | pad2 | 2 pad to align old SP
3767 // | +--------+ 1
3768 // | | locks | 0
3769 // | +--------+----> OptoReg::stack0(), even aligned
3770 // | | pad1 | 11 pad to align new SP
3771 // | +--------+
3772 // | | | 10
3773 // | | spills | 9 spills
3774 // V | | 8 (pad0 slot for callee)
3775 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3776 // ^ | out | 7
3777 // | | args | 6 Holes in outgoing args owned by CALLEE
3778 // Owned by +--------+
3779 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3780 // | new |preserve| Must be even-aligned.
3781 // | SP-+--------+----> Matcher::_new_SP, even aligned
3782 // | | |
3783 //
3784 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3785 // known from SELF's arguments and the Java calling convention.
3786 // Region 6-7 is determined per call site.
3787 // Note 2: If the calling convention leaves holes in the incoming argument
3788 // area, those holes are owned by SELF. Holes in the outgoing area
3789 // are owned by the CALLEE. Holes should not be necessary in the
3790 // incoming area, as the Java calling convention is completely under
3791 // the control of the AD file. Doubles can be sorted and packed to
3792 // avoid holes. Holes in the outgoing arguments may be necessary for
3793 // varargs C calling conventions.
3794 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3795 // even aligned with pad0 as needed.
3796 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3797 // (the latter is true on Intel but is it false on AArch64?)
3798 // region 6-11 is even aligned; it may be padded out more so that
3799 // the region from SP to FP meets the minimum stack alignment.
3800 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3801 // alignment. Region 11, pad1, may be dynamically extended so that
3802 // SP meets the minimum alignment.
3803
3804 frame %{
3805 // These three registers define part of the calling convention
3806 // between compiled code and the interpreter.
3807
3808 // Inline Cache Register or Method for I2C.
3809 inline_cache_reg(R12);
3810
3811 // Number of stack slots consumed by locking an object
3812 sync_stack_slots(2);
3813
3814 // Compiled code's Frame Pointer
3815 frame_pointer(R31);
3816
3817 // Interpreter stores its frame pointer in a register which is
3818 // stored to the stack by I2CAdaptors.
3819 // I2CAdaptors convert from interpreted java to compiled java.
3820 interpreter_frame_pointer(R29);
3821
3822 // Stack alignment requirement
3823 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3824
3825 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3826 // for calls to C. Supports the var-args backing area for register parms.
3827 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3828
3829 // The after-PROLOG location of the return address. Location of
3830 // return address specifies a type (REG or STACK) and a number
3831 // representing the register number (i.e. - use a register name) or
3832 // stack slot.
3833 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3834 // Otherwise, it is above the locks and verification slot and alignment word
3835 // TODO this may well be correct but need to check why that - 2 is there
3836 // ppc port uses 0 but we definitely need to allow for fixed_slots
3837 // which folds in the space used for monitors
3838 return_addr(STACK - 2 +
3839 align_up((Compile::current()->in_preserve_stack_slots() +
3840 Compile::current()->fixed_slots()),
3841 stack_alignment_in_slots()));
3842
3843 // Location of compiled Java return values. Same as C for now.
3844 return_value
3845 %{
3846 // TODO do we allow ideal_reg == Op_RegN???
3847 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3848 "only return normal values");
3849
3850 static const int lo[Op_RegL + 1] = { // enum name
3851 0, // Op_Node
3852 0, // Op_Set
3853 R0_num, // Op_RegN
3854 R0_num, // Op_RegI
3855 R0_num, // Op_RegP
3856 V0_num, // Op_RegF
3857 V0_num, // Op_RegD
3858 R0_num // Op_RegL
3859 };
3860
3861 static const int hi[Op_RegL + 1] = { // enum name
3862 0, // Op_Node
3863 0, // Op_Set
3864 OptoReg::Bad, // Op_RegN
3865 OptoReg::Bad, // Op_RegI
3866 R0_H_num, // Op_RegP
3867 OptoReg::Bad, // Op_RegF
3868 V0_H_num, // Op_RegD
3869 R0_H_num // Op_RegL
3870 };
3871
3872 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3873 %}
3874 %}
3875
3876 //----------ATTRIBUTES---------------------------------------------------------
3877 //----------Operand Attributes-------------------------------------------------
3878 op_attrib op_cost(1); // Required cost attribute
3879
3880 //----------Instruction Attributes---------------------------------------------
3881 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3882 ins_attrib ins_size(32); // Required size attribute (in bits)
3883 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3884 // a non-matching short branch variant
3885 // of some long branch?
3886 ins_attrib ins_alignment(4); // Required alignment attribute (must
3887 // be a power of 2) specifies the
3888 // alignment that some part of the
3889 // instruction (not necessarily the
3890 // start) requires. If > 1, a
3891 // compute_padding() function must be
3892 // provided for the instruction
3893
3894 // Whether this node is expanded during code emission into a sequence of
3895 // instructions and the first instruction can perform an implicit null check.
3896 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3897
3898 //----------OPERANDS-----------------------------------------------------------
3899 // Operand definitions must precede instruction definitions for correct parsing
3900 // in the ADLC because operands constitute user defined types which are used in
3901 // instruction definitions.
3902
3903 //----------Simple Operands----------------------------------------------------
3904
3905 // Integer operands 32 bit
3906 // 32 bit immediate
3907 operand immI()
3908 %{
3909 match(ConI);
3910
3911 op_cost(0);
3912 format %{ %}
3913 interface(CONST_INTER);
3914 %}
3915
3916 // 32 bit zero
3917 operand immI0()
3918 %{
3919 predicate(n->get_int() == 0);
3920 match(ConI);
3921
3922 op_cost(0);
3923 format %{ %}
3924 interface(CONST_INTER);
3925 %}
3926
3927 // 32 bit unit increment
3928 operand immI_1()
3929 %{
3930 predicate(n->get_int() == 1);
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 // 32 bit unit decrement
3939 operand immI_M1()
3940 %{
3941 predicate(n->get_int() == -1);
3942 match(ConI);
3943
3944 op_cost(0);
3945 format %{ %}
3946 interface(CONST_INTER);
3947 %}
3948
3949 // Shift values for add/sub extension shift
3950 operand immIExt()
3951 %{
3952 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3953 match(ConI);
3954
3955 op_cost(0);
3956 format %{ %}
3957 interface(CONST_INTER);
3958 %}
3959
3960 operand immI_gt_1()
3961 %{
3962 predicate(n->get_int() > 1);
3963 match(ConI);
3964
3965 op_cost(0);
3966 format %{ %}
3967 interface(CONST_INTER);
3968 %}
3969
3970 operand immI_le_4()
3971 %{
3972 predicate(n->get_int() <= 4);
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 operand immI_16()
3981 %{
3982 predicate(n->get_int() == 16);
3983 match(ConI);
3984
3985 op_cost(0);
3986 format %{ %}
3987 interface(CONST_INTER);
3988 %}
3989
3990 operand immI_24()
3991 %{
3992 predicate(n->get_int() == 24);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 operand immI_32()
4001 %{
4002 predicate(n->get_int() == 32);
4003 match(ConI);
4004
4005 op_cost(0);
4006 format %{ %}
4007 interface(CONST_INTER);
4008 %}
4009
4010 operand immI_48()
4011 %{
4012 predicate(n->get_int() == 48);
4013 match(ConI);
4014
4015 op_cost(0);
4016 format %{ %}
4017 interface(CONST_INTER);
4018 %}
4019
4020 operand immI_56()
4021 %{
4022 predicate(n->get_int() == 56);
4023 match(ConI);
4024
4025 op_cost(0);
4026 format %{ %}
4027 interface(CONST_INTER);
4028 %}
4029
4030 operand immI_255()
4031 %{
4032 predicate(n->get_int() == 255);
4033 match(ConI);
4034
4035 op_cost(0);
4036 format %{ %}
4037 interface(CONST_INTER);
4038 %}
4039
4040 operand immI_65535()
4041 %{
4042 predicate(n->get_int() == 65535);
4043 match(ConI);
4044
4045 op_cost(0);
4046 format %{ %}
4047 interface(CONST_INTER);
4048 %}
4049
4050 operand immI_positive()
4051 %{
4052 predicate(n->get_int() > 0);
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 // BoolTest condition for signed compare
4061 operand immI_cmp_cond()
4062 %{
4063 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4064 match(ConI);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 // BoolTest condition for unsigned compare
4072 operand immI_cmpU_cond()
4073 %{
4074 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4075 match(ConI);
4076
4077 op_cost(0);
4078 format %{ %}
4079 interface(CONST_INTER);
4080 %}
4081
4082 operand immL_255()
4083 %{
4084 predicate(n->get_long() == 255L);
4085 match(ConL);
4086
4087 op_cost(0);
4088 format %{ %}
4089 interface(CONST_INTER);
4090 %}
4091
4092 operand immL_65535()
4093 %{
4094 predicate(n->get_long() == 65535L);
4095 match(ConL);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 operand immL_4294967295()
4103 %{
4104 predicate(n->get_long() == 4294967295L);
4105 match(ConL);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immL_bitmask()
4113 %{
4114 predicate((n->get_long() != 0)
4115 && ((n->get_long() & 0xc000000000000000l) == 0)
4116 && is_power_of_2(n->get_long() + 1));
4117 match(ConL);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immI_bitmask()
4125 %{
4126 predicate((n->get_int() != 0)
4127 && ((n->get_int() & 0xc0000000) == 0)
4128 && is_power_of_2(n->get_int() + 1));
4129 match(ConI);
4130
4131 op_cost(0);
4132 format %{ %}
4133 interface(CONST_INTER);
4134 %}
4135
4136 operand immL_positive_bitmaskI()
4137 %{
4138 predicate((n->get_long() != 0)
4139 && ((julong)n->get_long() < 0x80000000ULL)
4140 && is_power_of_2(n->get_long() + 1));
4141 match(ConL);
4142
4143 op_cost(0);
4144 format %{ %}
4145 interface(CONST_INTER);
4146 %}
4147
4148 // Scale values for scaled offset addressing modes (up to long but not quad)
4149 operand immIScale()
4150 %{
4151 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4152 match(ConI);
4153
4154 op_cost(0);
4155 format %{ %}
4156 interface(CONST_INTER);
4157 %}
4158
4159 // 5 bit signed integer
4160 operand immI5()
4161 %{
4162 predicate(Assembler::is_simm(n->get_int(), 5));
4163 match(ConI);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 // 7 bit unsigned integer
4171 operand immIU7()
4172 %{
4173 predicate(Assembler::is_uimm(n->get_int(), 7));
4174 match(ConI);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 // Offset for scaled or unscaled immediate loads and stores
4182 operand immIOffset()
4183 %{
4184 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4185 match(ConI);
4186
4187 op_cost(0);
4188 format %{ %}
4189 interface(CONST_INTER);
4190 %}
4191
4192 operand immIOffset1()
4193 %{
4194 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4195 match(ConI);
4196
4197 op_cost(0);
4198 format %{ %}
4199 interface(CONST_INTER);
4200 %}
4201
4202 operand immIOffset2()
4203 %{
4204 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4205 match(ConI);
4206
4207 op_cost(0);
4208 format %{ %}
4209 interface(CONST_INTER);
4210 %}
4211
4212 operand immIOffset4()
4213 %{
4214 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4215 match(ConI);
4216
4217 op_cost(0);
4218 format %{ %}
4219 interface(CONST_INTER);
4220 %}
4221
4222 operand immIOffset8()
4223 %{
4224 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4225 match(ConI);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 operand immIOffset16()
4233 %{
4234 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4235 match(ConI);
4236
4237 op_cost(0);
4238 format %{ %}
4239 interface(CONST_INTER);
4240 %}
4241
4242 operand immLOffset()
4243 %{
4244 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4245 match(ConL);
4246
4247 op_cost(0);
4248 format %{ %}
4249 interface(CONST_INTER);
4250 %}
4251
4252 operand immLoffset1()
4253 %{
4254 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4255 match(ConL);
4256
4257 op_cost(0);
4258 format %{ %}
4259 interface(CONST_INTER);
4260 %}
4261
4262 operand immLoffset2()
4263 %{
4264 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4265 match(ConL);
4266
4267 op_cost(0);
4268 format %{ %}
4269 interface(CONST_INTER);
4270 %}
4271
4272 operand immLoffset4()
4273 %{
4274 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4275 match(ConL);
4276
4277 op_cost(0);
4278 format %{ %}
4279 interface(CONST_INTER);
4280 %}
4281
4282 operand immLoffset8()
4283 %{
4284 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4285 match(ConL);
4286
4287 op_cost(0);
4288 format %{ %}
4289 interface(CONST_INTER);
4290 %}
4291
4292 operand immLoffset16()
4293 %{
4294 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 // 5 bit signed long integer
4303 operand immL5()
4304 %{
4305 predicate(Assembler::is_simm(n->get_long(), 5));
4306 match(ConL);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 // 7 bit unsigned long integer
4314 operand immLU7()
4315 %{
4316 predicate(Assembler::is_uimm(n->get_long(), 7));
4317 match(ConL);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 8 bit signed value.
4325 operand immI8()
4326 %{
4327 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4328 match(ConI);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 8 bit signed value (simm8), or #simm8 LSL 8.
4336 operand immIDupV()
4337 %{
4338 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4339 match(ConI);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit signed value (simm8), or #simm8 LSL 8.
4347 operand immLDupV()
4348 %{
4349 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4350 match(ConL);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 // 8 bit signed value (simm8), or #simm8 LSL 8.
4358 operand immHDupV()
4359 %{
4360 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4361 match(ConH);
4362
4363 op_cost(0);
4364 format %{ %}
4365 interface(CONST_INTER);
4366 %}
4367
4368 // 8 bit integer valid for vector add sub immediate
4369 operand immBAddSubV()
4370 %{
4371 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4372 match(ConI);
4373
4374 op_cost(0);
4375 format %{ %}
4376 interface(CONST_INTER);
4377 %}
4378
4379 // 32 bit integer valid for add sub immediate
4380 operand immIAddSub()
4381 %{
4382 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4383 match(ConI);
4384 op_cost(0);
4385 format %{ %}
4386 interface(CONST_INTER);
4387 %}
4388
4389 // 32 bit integer valid for vector add sub immediate
4390 operand immIAddSubV()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4393 match(ConI);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 // 32 bit unsigned integer valid for logical immediate
4401
4402 operand immBLog()
4403 %{
4404 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4405 match(ConI);
4406
4407 op_cost(0);
4408 format %{ %}
4409 interface(CONST_INTER);
4410 %}
4411
4412 operand immSLog()
4413 %{
4414 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4415 match(ConI);
4416
4417 op_cost(0);
4418 format %{ %}
4419 interface(CONST_INTER);
4420 %}
4421
4422 operand immILog()
4423 %{
4424 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4425 match(ConI);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 // Integer operands 64 bit
4433 // 64 bit immediate
4434 operand immL()
4435 %{
4436 match(ConL);
4437
4438 op_cost(0);
4439 format %{ %}
4440 interface(CONST_INTER);
4441 %}
4442
4443 // 64 bit zero
4444 operand immL0()
4445 %{
4446 predicate(n->get_long() == 0);
4447 match(ConL);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // 64 bit unit decrement
4455 operand immL_M1()
4456 %{
4457 predicate(n->get_long() == -1);
4458 match(ConL);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 // 64 bit integer valid for add sub immediate
4466 operand immLAddSub()
4467 %{
4468 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4469 match(ConL);
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 // 64 bit integer valid for addv subv immediate
4476 operand immLAddSubV()
4477 %{
4478 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4479 match(ConL);
4480
4481 op_cost(0);
4482 format %{ %}
4483 interface(CONST_INTER);
4484 %}
4485
4486 // 64 bit integer valid for logical immediate
4487 operand immLLog()
4488 %{
4489 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4490 match(ConL);
4491 op_cost(0);
4492 format %{ %}
4493 interface(CONST_INTER);
4494 %}
4495
4496 // Long Immediate: low 32-bit mask
4497 operand immL_32bits()
4498 %{
4499 predicate(n->get_long() == 0xFFFFFFFFL);
4500 match(ConL);
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 // Pointer operands
4507 // Pointer Immediate
4508 operand immP()
4509 %{
4510 match(ConP);
4511
4512 op_cost(0);
4513 format %{ %}
4514 interface(CONST_INTER);
4515 %}
4516
4517 // nullptr Pointer Immediate
4518 operand immP0()
4519 %{
4520 predicate(n->get_ptr() == 0);
4521 match(ConP);
4522
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // Pointer Immediate One
4529 // this is used in object initialization (initial object header)
4530 operand immP_1()
4531 %{
4532 predicate(n->get_ptr() == 1);
4533 match(ConP);
4534
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // AOT Runtime Constants Address
4541 operand immAOTRuntimeConstantsAddress()
4542 %{
4543 // Check if the address is in the range of AOT Runtime Constants
4544 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4545 match(ConP);
4546
4547 op_cost(0);
4548 format %{ %}
4549 interface(CONST_INTER);
4550 %}
4551
4552 // Float and Double operands
4553 // Double Immediate
4554 operand immD()
4555 %{
4556 match(ConD);
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 // Double Immediate: +0.0d
4563 operand immD0()
4564 %{
4565 predicate(jlong_cast(n->getd()) == 0);
4566 match(ConD);
4567
4568 op_cost(0);
4569 format %{ %}
4570 interface(CONST_INTER);
4571 %}
4572
4573 // constant 'double +0.0'.
4574 operand immDPacked()
4575 %{
4576 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4577 match(ConD);
4578 op_cost(0);
4579 format %{ %}
4580 interface(CONST_INTER);
4581 %}
4582
4583 // Float Immediate
4584 operand immF()
4585 %{
4586 match(ConF);
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 // Float Immediate: +0.0f.
4593 operand immF0()
4594 %{
4595 predicate(jint_cast(n->getf()) == 0);
4596 match(ConF);
4597
4598 op_cost(0);
4599 format %{ %}
4600 interface(CONST_INTER);
4601 %}
4602
4603 // Half Float (FP16) Immediate
4604 operand immH()
4605 %{
4606 match(ConH);
4607 op_cost(0);
4608 format %{ %}
4609 interface(CONST_INTER);
4610 %}
4611
4612 //
4613 operand immFPacked()
4614 %{
4615 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4616 match(ConF);
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 // Narrow pointer operands
4623 // Narrow Pointer Immediate
4624 operand immN()
4625 %{
4626 match(ConN);
4627
4628 op_cost(0);
4629 format %{ %}
4630 interface(CONST_INTER);
4631 %}
4632
4633 // Narrow nullptr Pointer Immediate
4634 operand immN0()
4635 %{
4636 predicate(n->get_narrowcon() == 0);
4637 match(ConN);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 operand immNKlass()
4645 %{
4646 match(ConNKlass);
4647
4648 op_cost(0);
4649 format %{ %}
4650 interface(CONST_INTER);
4651 %}
4652
4653 // Integer 32 bit Register Operands
4654 // Integer 32 bitRegister (excludes SP)
4655 operand iRegI()
4656 %{
4657 constraint(ALLOC_IN_RC(any_reg32));
4658 match(RegI);
4659 match(iRegINoSp);
4660 op_cost(0);
4661 format %{ %}
4662 interface(REG_INTER);
4663 %}
4664
4665 // Integer 32 bit Register not Special
4666 operand iRegINoSp()
4667 %{
4668 constraint(ALLOC_IN_RC(no_special_reg32));
4669 match(RegI);
4670 op_cost(0);
4671 format %{ %}
4672 interface(REG_INTER);
4673 %}
4674
4675 // Integer 64 bit Register Operands
4676 // Integer 64 bit Register (includes SP)
4677 operand iRegL()
4678 %{
4679 constraint(ALLOC_IN_RC(any_reg));
4680 match(RegL);
4681 match(iRegLNoSp);
4682 op_cost(0);
4683 format %{ %}
4684 interface(REG_INTER);
4685 %}
4686
4687 // Integer 64 bit Register not Special
4688 operand iRegLNoSp()
4689 %{
4690 constraint(ALLOC_IN_RC(no_special_reg));
4691 match(RegL);
4692 match(iRegL_R0);
4693 format %{ %}
4694 interface(REG_INTER);
4695 %}
4696
4697 // Pointer Register Operands
4698 // Pointer Register
4699 operand iRegP()
4700 %{
4701 constraint(ALLOC_IN_RC(ptr_reg));
4702 match(RegP);
4703 match(iRegPNoSp);
4704 match(iRegP_R0);
4705 //match(iRegP_R2);
4706 //match(iRegP_R4);
4707 match(iRegP_R5);
4708 match(thread_RegP);
4709 op_cost(0);
4710 format %{ %}
4711 interface(REG_INTER);
4712 %}
4713
4714 // Pointer 64 bit Register not Special
4715 operand iRegPNoSp()
4716 %{
4717 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4718 match(RegP);
4719 // match(iRegP);
4720 // match(iRegP_R0);
4721 // match(iRegP_R2);
4722 // match(iRegP_R4);
4723 // match(iRegP_R5);
4724 // match(thread_RegP);
4725 op_cost(0);
4726 format %{ %}
4727 interface(REG_INTER);
4728 %}
4729
4730 // This operand is not allowed to use rfp even if
4731 // rfp is not used to hold the frame pointer.
4732 operand iRegPNoSpNoRfp()
4733 %{
4734 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4735 match(RegP);
4736 match(iRegPNoSp);
4737 op_cost(0);
4738 format %{ %}
4739 interface(REG_INTER);
4740 %}
4741
4742 // Pointer 64 bit Register R0 only
4743 operand iRegP_R0()
4744 %{
4745 constraint(ALLOC_IN_RC(r0_reg));
4746 match(RegP);
4747 // match(iRegP);
4748 match(iRegPNoSp);
4749 op_cost(0);
4750 format %{ %}
4751 interface(REG_INTER);
4752 %}
4753
4754 // Pointer 64 bit Register R1 only
4755 operand iRegP_R1()
4756 %{
4757 constraint(ALLOC_IN_RC(r1_reg));
4758 match(RegP);
4759 // match(iRegP);
4760 match(iRegPNoSp);
4761 op_cost(0);
4762 format %{ %}
4763 interface(REG_INTER);
4764 %}
4765
4766 // Pointer 64 bit Register R2 only
4767 operand iRegP_R2()
4768 %{
4769 constraint(ALLOC_IN_RC(r2_reg));
4770 match(RegP);
4771 // match(iRegP);
4772 match(iRegPNoSp);
4773 op_cost(0);
4774 format %{ %}
4775 interface(REG_INTER);
4776 %}
4777
4778 // Pointer 64 bit Register R3 only
4779 operand iRegP_R3()
4780 %{
4781 constraint(ALLOC_IN_RC(r3_reg));
4782 match(RegP);
4783 // match(iRegP);
4784 match(iRegPNoSp);
4785 op_cost(0);
4786 format %{ %}
4787 interface(REG_INTER);
4788 %}
4789
4790 // Pointer 64 bit Register R4 only
4791 operand iRegP_R4()
4792 %{
4793 constraint(ALLOC_IN_RC(r4_reg));
4794 match(RegP);
4795 // match(iRegP);
4796 match(iRegPNoSp);
4797 op_cost(0);
4798 format %{ %}
4799 interface(REG_INTER);
4800 %}
4801
4802 // Pointer 64 bit Register R5 only
4803 operand iRegP_R5()
4804 %{
4805 constraint(ALLOC_IN_RC(r5_reg));
4806 match(RegP);
4807 // match(iRegP);
4808 match(iRegPNoSp);
4809 op_cost(0);
4810 format %{ %}
4811 interface(REG_INTER);
4812 %}
4813
4814 // Pointer 64 bit Register R10 only
4815 operand iRegP_R10()
4816 %{
4817 constraint(ALLOC_IN_RC(r10_reg));
4818 match(RegP);
4819 // match(iRegP);
4820 match(iRegPNoSp);
4821 op_cost(0);
4822 format %{ %}
4823 interface(REG_INTER);
4824 %}
4825
4826 // Long 64 bit Register R0 only
4827 operand iRegL_R0()
4828 %{
4829 constraint(ALLOC_IN_RC(r0_reg));
4830 match(RegL);
4831 match(iRegLNoSp);
4832 op_cost(0);
4833 format %{ %}
4834 interface(REG_INTER);
4835 %}
4836
4837 // Long 64 bit Register R11 only
4838 operand iRegL_R11()
4839 %{
4840 constraint(ALLOC_IN_RC(r11_reg));
4841 match(RegL);
4842 match(iRegLNoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Register R0 only
4849 operand iRegI_R0()
4850 %{
4851 constraint(ALLOC_IN_RC(int_r0_reg));
4852 match(RegI);
4853 match(iRegINoSp);
4854 op_cost(0);
4855 format %{ %}
4856 interface(REG_INTER);
4857 %}
4858
4859 // Register R2 only
4860 operand iRegI_R2()
4861 %{
4862 constraint(ALLOC_IN_RC(int_r2_reg));
4863 match(RegI);
4864 match(iRegINoSp);
4865 op_cost(0);
4866 format %{ %}
4867 interface(REG_INTER);
4868 %}
4869
4870 // Register R3 only
4871 operand iRegI_R3()
4872 %{
4873 constraint(ALLOC_IN_RC(int_r3_reg));
4874 match(RegI);
4875 match(iRegINoSp);
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881
4882 // Register R4 only
4883 operand iRegI_R4()
4884 %{
4885 constraint(ALLOC_IN_RC(int_r4_reg));
4886 match(RegI);
4887 match(iRegINoSp);
4888 op_cost(0);
4889 format %{ %}
4890 interface(REG_INTER);
4891 %}
4892
4893
4894 // Pointer Register Operands
4895 // Narrow Pointer Register
4896 operand iRegN()
4897 %{
4898 constraint(ALLOC_IN_RC(any_reg32));
4899 match(RegN);
4900 match(iRegNNoSp);
4901 op_cost(0);
4902 format %{ %}
4903 interface(REG_INTER);
4904 %}
4905
4906 // Integer 64 bit Register not Special
4907 operand iRegNNoSp()
4908 %{
4909 constraint(ALLOC_IN_RC(no_special_reg32));
4910 match(RegN);
4911 op_cost(0);
4912 format %{ %}
4913 interface(REG_INTER);
4914 %}
4915
4916 // Float Register
4917 // Float register operands
4918 operand vRegF()
4919 %{
4920 constraint(ALLOC_IN_RC(float_reg));
4921 match(RegF);
4922
4923 op_cost(0);
4924 format %{ %}
4925 interface(REG_INTER);
4926 %}
4927
4928 // Double Register
4929 // Double register operands
4930 operand vRegD()
4931 %{
4932 constraint(ALLOC_IN_RC(double_reg));
4933 match(RegD);
4934
4935 op_cost(0);
4936 format %{ %}
4937 interface(REG_INTER);
4938 %}
4939
4940 // Generic vector class. This will be used for
4941 // all vector operands, including NEON and SVE.
4942 operand vReg()
4943 %{
4944 constraint(ALLOC_IN_RC(dynamic));
4945 match(VecA);
4946 match(VecD);
4947 match(VecX);
4948
4949 op_cost(0);
4950 format %{ %}
4951 interface(REG_INTER);
4952 %}
4953
4954 operand vReg_V10()
4955 %{
4956 constraint(ALLOC_IN_RC(v10_veca_reg));
4957 match(vReg);
4958
4959 op_cost(0);
4960 format %{ %}
4961 interface(REG_INTER);
4962 %}
4963
4964 operand vReg_V11()
4965 %{
4966 constraint(ALLOC_IN_RC(v11_veca_reg));
4967 match(vReg);
4968
4969 op_cost(0);
4970 format %{ %}
4971 interface(REG_INTER);
4972 %}
4973
4974 operand vReg_V12()
4975 %{
4976 constraint(ALLOC_IN_RC(v12_veca_reg));
4977 match(vReg);
4978
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 operand vReg_V13()
4985 %{
4986 constraint(ALLOC_IN_RC(v13_veca_reg));
4987 match(vReg);
4988
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 operand vReg_V17()
4995 %{
4996 constraint(ALLOC_IN_RC(v17_veca_reg));
4997 match(vReg);
4998
4999 op_cost(0);
5000 format %{ %}
5001 interface(REG_INTER);
5002 %}
5003
5004 operand vReg_V18()
5005 %{
5006 constraint(ALLOC_IN_RC(v18_veca_reg));
5007 match(vReg);
5008
5009 op_cost(0);
5010 format %{ %}
5011 interface(REG_INTER);
5012 %}
5013
5014 operand vReg_V23()
5015 %{
5016 constraint(ALLOC_IN_RC(v23_veca_reg));
5017 match(vReg);
5018
5019 op_cost(0);
5020 format %{ %}
5021 interface(REG_INTER);
5022 %}
5023
5024 operand vReg_V24()
5025 %{
5026 constraint(ALLOC_IN_RC(v24_veca_reg));
5027 match(vReg);
5028
5029 op_cost(0);
5030 format %{ %}
5031 interface(REG_INTER);
5032 %}
5033
5034 operand vecA()
5035 %{
5036 constraint(ALLOC_IN_RC(vectora_reg));
5037 match(VecA);
5038
5039 op_cost(0);
5040 format %{ %}
5041 interface(REG_INTER);
5042 %}
5043
5044 operand vecD()
5045 %{
5046 constraint(ALLOC_IN_RC(vectord_reg));
5047 match(VecD);
5048
5049 op_cost(0);
5050 format %{ %}
5051 interface(REG_INTER);
5052 %}
5053
5054 operand vecX()
5055 %{
5056 constraint(ALLOC_IN_RC(vectorx_reg));
5057 match(VecX);
5058
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vRegD_V0()
5065 %{
5066 constraint(ALLOC_IN_RC(v0_reg));
5067 match(RegD);
5068 op_cost(0);
5069 format %{ %}
5070 interface(REG_INTER);
5071 %}
5072
5073 operand vRegD_V1()
5074 %{
5075 constraint(ALLOC_IN_RC(v1_reg));
5076 match(RegD);
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vRegD_V2()
5083 %{
5084 constraint(ALLOC_IN_RC(v2_reg));
5085 match(RegD);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vRegD_V3()
5092 %{
5093 constraint(ALLOC_IN_RC(v3_reg));
5094 match(RegD);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vRegD_V4()
5101 %{
5102 constraint(ALLOC_IN_RC(v4_reg));
5103 match(RegD);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand vRegD_V5()
5110 %{
5111 constraint(ALLOC_IN_RC(v5_reg));
5112 match(RegD);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand vRegD_V6()
5119 %{
5120 constraint(ALLOC_IN_RC(v6_reg));
5121 match(RegD);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 operand vRegD_V7()
5128 %{
5129 constraint(ALLOC_IN_RC(v7_reg));
5130 match(RegD);
5131 op_cost(0);
5132 format %{ %}
5133 interface(REG_INTER);
5134 %}
5135
5136 operand vRegD_V12()
5137 %{
5138 constraint(ALLOC_IN_RC(v12_reg));
5139 match(RegD);
5140 op_cost(0);
5141 format %{ %}
5142 interface(REG_INTER);
5143 %}
5144
5145 operand vRegD_V13()
5146 %{
5147 constraint(ALLOC_IN_RC(v13_reg));
5148 match(RegD);
5149 op_cost(0);
5150 format %{ %}
5151 interface(REG_INTER);
5152 %}
5153
5154 operand pReg()
5155 %{
5156 constraint(ALLOC_IN_RC(pr_reg));
5157 match(RegVectMask);
5158 match(pRegGov);
5159 op_cost(0);
5160 format %{ %}
5161 interface(REG_INTER);
5162 %}
5163
5164 operand pRegGov()
5165 %{
5166 constraint(ALLOC_IN_RC(gov_pr));
5167 match(RegVectMask);
5168 match(pReg);
5169 op_cost(0);
5170 format %{ %}
5171 interface(REG_INTER);
5172 %}
5173
5174 operand pRegGov_P0()
5175 %{
5176 constraint(ALLOC_IN_RC(p0_reg));
5177 match(RegVectMask);
5178 op_cost(0);
5179 format %{ %}
5180 interface(REG_INTER);
5181 %}
5182
5183 operand pRegGov_P1()
5184 %{
5185 constraint(ALLOC_IN_RC(p1_reg));
5186 match(RegVectMask);
5187 op_cost(0);
5188 format %{ %}
5189 interface(REG_INTER);
5190 %}
5191
5192 // Flags register, used as output of signed compare instructions
5193
5194 // note that on AArch64 we also use this register as the output for
5195 // for floating point compare instructions (CmpF CmpD). this ensures
5196 // that ordered inequality tests use GT, GE, LT or LE none of which
5197 // pass through cases where the result is unordered i.e. one or both
5198 // inputs to the compare is a NaN. this means that the ideal code can
5199 // replace e.g. a GT with an LE and not end up capturing the NaN case
5200 // (where the comparison should always fail). EQ and NE tests are
5201 // always generated in ideal code so that unordered folds into the NE
5202 // case, matching the behaviour of AArch64 NE.
5203 //
5204 // This differs from x86 where the outputs of FP compares use a
5205 // special FP flags registers and where compares based on this
5206 // register are distinguished into ordered inequalities (cmpOpUCF) and
5207 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5208 // to explicitly handle the unordered case in branches. x86 also has
5209 // to include extra CMoveX rules to accept a cmpOpUCF input.
5210
5211 operand rFlagsReg()
5212 %{
5213 constraint(ALLOC_IN_RC(int_flags));
5214 match(RegFlags);
5215
5216 op_cost(0);
5217 format %{ "RFLAGS" %}
5218 interface(REG_INTER);
5219 %}
5220
5221 // Flags register, used as output of unsigned compare instructions
5222 operand rFlagsRegU()
5223 %{
5224 constraint(ALLOC_IN_RC(int_flags));
5225 match(RegFlags);
5226
5227 op_cost(0);
5228 format %{ "RFLAGSU" %}
5229 interface(REG_INTER);
5230 %}
5231
5232 // Special Registers
5233
5234 // Method Register
5235 operand inline_cache_RegP(iRegP reg)
5236 %{
5237 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5238 match(reg);
5239 match(iRegPNoSp);
5240 op_cost(0);
5241 format %{ %}
5242 interface(REG_INTER);
5243 %}
5244
5245 // Thread Register
5246 operand thread_RegP(iRegP reg)
5247 %{
5248 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5249 match(reg);
5250 op_cost(0);
5251 format %{ %}
5252 interface(REG_INTER);
5253 %}
5254
5255 //----------Memory Operands----------------------------------------------------
5256
5257 operand indirect(iRegP reg)
5258 %{
5259 constraint(ALLOC_IN_RC(ptr_reg));
5260 match(reg);
5261 op_cost(0);
5262 format %{ "[$reg]" %}
5263 interface(MEMORY_INTER) %{
5264 base($reg);
5265 index(0xffffffff);
5266 scale(0x0);
5267 disp(0x0);
5268 %}
5269 %}
5270
5271 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5272 %{
5273 constraint(ALLOC_IN_RC(ptr_reg));
5274 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5275 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5276 op_cost(0);
5277 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5278 interface(MEMORY_INTER) %{
5279 base($reg);
5280 index($ireg);
5281 scale($scale);
5282 disp(0x0);
5283 %}
5284 %}
5285
5286 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5287 %{
5288 constraint(ALLOC_IN_RC(ptr_reg));
5289 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5290 match(AddP reg (LShiftL lreg scale));
5291 op_cost(0);
5292 format %{ "$reg, $lreg lsl($scale)" %}
5293 interface(MEMORY_INTER) %{
5294 base($reg);
5295 index($lreg);
5296 scale($scale);
5297 disp(0x0);
5298 %}
5299 %}
5300
5301 operand indIndexI2L(iRegP reg, iRegI ireg)
5302 %{
5303 constraint(ALLOC_IN_RC(ptr_reg));
5304 match(AddP reg (ConvI2L ireg));
5305 op_cost(0);
5306 format %{ "$reg, $ireg, 0, I2L" %}
5307 interface(MEMORY_INTER) %{
5308 base($reg);
5309 index($ireg);
5310 scale(0x0);
5311 disp(0x0);
5312 %}
5313 %}
5314
5315 operand indIndex(iRegP reg, iRegL lreg)
5316 %{
5317 constraint(ALLOC_IN_RC(ptr_reg));
5318 match(AddP reg lreg);
5319 op_cost(0);
5320 format %{ "$reg, $lreg" %}
5321 interface(MEMORY_INTER) %{
5322 base($reg);
5323 index($lreg);
5324 scale(0x0);
5325 disp(0x0);
5326 %}
5327 %}
5328
5329 operand indOffI1(iRegP reg, immIOffset1 off)
5330 %{
5331 constraint(ALLOC_IN_RC(ptr_reg));
5332 match(AddP reg off);
5333 op_cost(0);
5334 format %{ "[$reg, $off]" %}
5335 interface(MEMORY_INTER) %{
5336 base($reg);
5337 index(0xffffffff);
5338 scale(0x0);
5339 disp($off);
5340 %}
5341 %}
5342
5343 operand indOffI2(iRegP reg, immIOffset2 off)
5344 %{
5345 constraint(ALLOC_IN_RC(ptr_reg));
5346 match(AddP reg off);
5347 op_cost(0);
5348 format %{ "[$reg, $off]" %}
5349 interface(MEMORY_INTER) %{
5350 base($reg);
5351 index(0xffffffff);
5352 scale(0x0);
5353 disp($off);
5354 %}
5355 %}
5356
5357 operand indOffI4(iRegP reg, immIOffset4 off)
5358 %{
5359 constraint(ALLOC_IN_RC(ptr_reg));
5360 match(AddP reg off);
5361 op_cost(0);
5362 format %{ "[$reg, $off]" %}
5363 interface(MEMORY_INTER) %{
5364 base($reg);
5365 index(0xffffffff);
5366 scale(0x0);
5367 disp($off);
5368 %}
5369 %}
5370
5371 operand indOffI8(iRegP reg, immIOffset8 off)
5372 %{
5373 constraint(ALLOC_IN_RC(ptr_reg));
5374 match(AddP reg off);
5375 op_cost(0);
5376 format %{ "[$reg, $off]" %}
5377 interface(MEMORY_INTER) %{
5378 base($reg);
5379 index(0xffffffff);
5380 scale(0x0);
5381 disp($off);
5382 %}
5383 %}
5384
5385 operand indOffI16(iRegP reg, immIOffset16 off)
5386 %{
5387 constraint(ALLOC_IN_RC(ptr_reg));
5388 match(AddP reg off);
5389 op_cost(0);
5390 format %{ "[$reg, $off]" %}
5391 interface(MEMORY_INTER) %{
5392 base($reg);
5393 index(0xffffffff);
5394 scale(0x0);
5395 disp($off);
5396 %}
5397 %}
5398
5399 operand indOffL1(iRegP reg, immLoffset1 off)
5400 %{
5401 constraint(ALLOC_IN_RC(ptr_reg));
5402 match(AddP reg off);
5403 op_cost(0);
5404 format %{ "[$reg, $off]" %}
5405 interface(MEMORY_INTER) %{
5406 base($reg);
5407 index(0xffffffff);
5408 scale(0x0);
5409 disp($off);
5410 %}
5411 %}
5412
5413 operand indOffL2(iRegP reg, immLoffset2 off)
5414 %{
5415 constraint(ALLOC_IN_RC(ptr_reg));
5416 match(AddP reg off);
5417 op_cost(0);
5418 format %{ "[$reg, $off]" %}
5419 interface(MEMORY_INTER) %{
5420 base($reg);
5421 index(0xffffffff);
5422 scale(0x0);
5423 disp($off);
5424 %}
5425 %}
5426
5427 operand indOffL4(iRegP reg, immLoffset4 off)
5428 %{
5429 constraint(ALLOC_IN_RC(ptr_reg));
5430 match(AddP reg off);
5431 op_cost(0);
5432 format %{ "[$reg, $off]" %}
5433 interface(MEMORY_INTER) %{
5434 base($reg);
5435 index(0xffffffff);
5436 scale(0x0);
5437 disp($off);
5438 %}
5439 %}
5440
5441 operand indOffL8(iRegP reg, immLoffset8 off)
5442 %{
5443 constraint(ALLOC_IN_RC(ptr_reg));
5444 match(AddP reg off);
5445 op_cost(0);
5446 format %{ "[$reg, $off]" %}
5447 interface(MEMORY_INTER) %{
5448 base($reg);
5449 index(0xffffffff);
5450 scale(0x0);
5451 disp($off);
5452 %}
5453 %}
5454
5455 operand indOffL16(iRegP reg, immLoffset16 off)
5456 %{
5457 constraint(ALLOC_IN_RC(ptr_reg));
5458 match(AddP reg off);
5459 op_cost(0);
5460 format %{ "[$reg, $off]" %}
5461 interface(MEMORY_INTER) %{
5462 base($reg);
5463 index(0xffffffff);
5464 scale(0x0);
5465 disp($off);
5466 %}
5467 %}
5468
5469 operand indirectX2P(iRegL reg)
5470 %{
5471 constraint(ALLOC_IN_RC(ptr_reg));
5472 match(CastX2P reg);
5473 op_cost(0);
5474 format %{ "[$reg]\t# long -> ptr" %}
5475 interface(MEMORY_INTER) %{
5476 base($reg);
5477 index(0xffffffff);
5478 scale(0x0);
5479 disp(0x0);
5480 %}
5481 %}
5482
5483 operand indOffX2P(iRegL reg, immLOffset off)
5484 %{
5485 constraint(ALLOC_IN_RC(ptr_reg));
5486 match(AddP (CastX2P reg) off);
5487 op_cost(0);
5488 format %{ "[$reg, $off]\t# long -> ptr" %}
5489 interface(MEMORY_INTER) %{
5490 base($reg);
5491 index(0xffffffff);
5492 scale(0x0);
5493 disp($off);
5494 %}
5495 %}
5496
5497 operand indirectN(iRegN reg)
5498 %{
5499 predicate(CompressedOops::shift() == 0);
5500 constraint(ALLOC_IN_RC(ptr_reg));
5501 match(DecodeN reg);
5502 op_cost(0);
5503 format %{ "[$reg]\t# narrow" %}
5504 interface(MEMORY_INTER) %{
5505 base($reg);
5506 index(0xffffffff);
5507 scale(0x0);
5508 disp(0x0);
5509 %}
5510 %}
5511
5512 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5513 %{
5514 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5515 constraint(ALLOC_IN_RC(ptr_reg));
5516 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5517 op_cost(0);
5518 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5519 interface(MEMORY_INTER) %{
5520 base($reg);
5521 index($ireg);
5522 scale($scale);
5523 disp(0x0);
5524 %}
5525 %}
5526
5527 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5528 %{
5529 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5530 constraint(ALLOC_IN_RC(ptr_reg));
5531 match(AddP (DecodeN reg) (LShiftL lreg scale));
5532 op_cost(0);
5533 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5534 interface(MEMORY_INTER) %{
5535 base($reg);
5536 index($lreg);
5537 scale($scale);
5538 disp(0x0);
5539 %}
5540 %}
5541
5542 operand indIndexI2LN(iRegN reg, iRegI ireg)
5543 %{
5544 predicate(CompressedOops::shift() == 0);
5545 constraint(ALLOC_IN_RC(ptr_reg));
5546 match(AddP (DecodeN reg) (ConvI2L ireg));
5547 op_cost(0);
5548 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5549 interface(MEMORY_INTER) %{
5550 base($reg);
5551 index($ireg);
5552 scale(0x0);
5553 disp(0x0);
5554 %}
5555 %}
5556
5557 operand indIndexN(iRegN reg, iRegL lreg)
5558 %{
5559 predicate(CompressedOops::shift() == 0);
5560 constraint(ALLOC_IN_RC(ptr_reg));
5561 match(AddP (DecodeN reg) lreg);
5562 op_cost(0);
5563 format %{ "$reg, $lreg\t# narrow" %}
5564 interface(MEMORY_INTER) %{
5565 base($reg);
5566 index($lreg);
5567 scale(0x0);
5568 disp(0x0);
5569 %}
5570 %}
5571
5572 operand indOffIN(iRegN reg, immIOffset off)
5573 %{
5574 predicate(CompressedOops::shift() == 0);
5575 constraint(ALLOC_IN_RC(ptr_reg));
5576 match(AddP (DecodeN reg) off);
5577 op_cost(0);
5578 format %{ "[$reg, $off]\t# narrow" %}
5579 interface(MEMORY_INTER) %{
5580 base($reg);
5581 index(0xffffffff);
5582 scale(0x0);
5583 disp($off);
5584 %}
5585 %}
5586
5587 operand indOffLN(iRegN reg, immLOffset off)
5588 %{
5589 predicate(CompressedOops::shift() == 0);
5590 constraint(ALLOC_IN_RC(ptr_reg));
5591 match(AddP (DecodeN reg) off);
5592 op_cost(0);
5593 format %{ "[$reg, $off]\t# narrow" %}
5594 interface(MEMORY_INTER) %{
5595 base($reg);
5596 index(0xffffffff);
5597 scale(0x0);
5598 disp($off);
5599 %}
5600 %}
5601
5602
5603 //----------Special Memory Operands--------------------------------------------
5604 // Stack Slot Operand - This operand is used for loading and storing temporary
5605 // values on the stack where a match requires a value to
5606 // flow through memory.
5607 operand stackSlotP(sRegP reg)
5608 %{
5609 constraint(ALLOC_IN_RC(stack_slots));
5610 op_cost(100);
5611 // No match rule because this operand is only generated in matching
5612 // match(RegP);
5613 format %{ "[$reg]" %}
5614 interface(MEMORY_INTER) %{
5615 base(0x1e); // RSP
5616 index(0x0); // No Index
5617 scale(0x0); // No Scale
5618 disp($reg); // Stack Offset
5619 %}
5620 %}
5621
5622 operand stackSlotI(sRegI reg)
5623 %{
5624 constraint(ALLOC_IN_RC(stack_slots));
5625 // No match rule because this operand is only generated in matching
5626 // match(RegI);
5627 format %{ "[$reg]" %}
5628 interface(MEMORY_INTER) %{
5629 base(0x1e); // RSP
5630 index(0x0); // No Index
5631 scale(0x0); // No Scale
5632 disp($reg); // Stack Offset
5633 %}
5634 %}
5635
5636 operand stackSlotF(sRegF reg)
5637 %{
5638 constraint(ALLOC_IN_RC(stack_slots));
5639 // No match rule because this operand is only generated in matching
5640 // match(RegF);
5641 format %{ "[$reg]" %}
5642 interface(MEMORY_INTER) %{
5643 base(0x1e); // RSP
5644 index(0x0); // No Index
5645 scale(0x0); // No Scale
5646 disp($reg); // Stack Offset
5647 %}
5648 %}
5649
5650 operand stackSlotD(sRegD reg)
5651 %{
5652 constraint(ALLOC_IN_RC(stack_slots));
5653 // No match rule because this operand is only generated in matching
5654 // match(RegD);
5655 format %{ "[$reg]" %}
5656 interface(MEMORY_INTER) %{
5657 base(0x1e); // RSP
5658 index(0x0); // No Index
5659 scale(0x0); // No Scale
5660 disp($reg); // Stack Offset
5661 %}
5662 %}
5663
5664 operand stackSlotL(sRegL reg)
5665 %{
5666 constraint(ALLOC_IN_RC(stack_slots));
5667 // No match rule because this operand is only generated in matching
5668 // match(RegL);
5669 format %{ "[$reg]" %}
5670 interface(MEMORY_INTER) %{
5671 base(0x1e); // RSP
5672 index(0x0); // No Index
5673 scale(0x0); // No Scale
5674 disp($reg); // Stack Offset
5675 %}
5676 %}
5677
5678 // Operands for expressing Control Flow
5679 // NOTE: Label is a predefined operand which should not be redefined in
5680 // the AD file. It is generically handled within the ADLC.
5681
5682 //----------Conditional Branch Operands----------------------------------------
5683 // Comparison Op - This is the operation of the comparison, and is limited to
5684 // the following set of codes:
5685 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5686 //
5687 // Other attributes of the comparison, such as unsignedness, are specified
5688 // by the comparison instruction that sets a condition code flags register.
5689 // That result is represented by a flags operand whose subtype is appropriate
5690 // to the unsignedness (etc.) of the comparison.
5691 //
5692 // Later, the instruction which matches both the Comparison Op (a Bool) and
5693 // the flags (produced by the Cmp) specifies the coding of the comparison op
5694 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5695
5696 // used for signed integral comparisons and fp comparisons
5697
5698 operand cmpOp()
5699 %{
5700 match(Bool);
5701
5702 format %{ "" %}
5703 interface(COND_INTER) %{
5704 equal(0x0, "eq");
5705 not_equal(0x1, "ne");
5706 less(0xb, "lt");
5707 greater_equal(0xa, "ge");
5708 less_equal(0xd, "le");
5709 greater(0xc, "gt");
5710 overflow(0x6, "vs");
5711 no_overflow(0x7, "vc");
5712 %}
5713 %}
5714
5715 // used for unsigned integral comparisons
5716
5717 operand cmpOpU()
5718 %{
5719 match(Bool);
5720
5721 format %{ "" %}
5722 interface(COND_INTER) %{
5723 equal(0x0, "eq");
5724 not_equal(0x1, "ne");
5725 less(0x3, "lo");
5726 greater_equal(0x2, "hs");
5727 less_equal(0x9, "ls");
5728 greater(0x8, "hi");
5729 overflow(0x6, "vs");
5730 no_overflow(0x7, "vc");
5731 %}
5732 %}
5733
5734 // used for certain integral comparisons which can be
5735 // converted to cbxx or tbxx instructions
5736
5737 operand cmpOpEqNe()
5738 %{
5739 match(Bool);
5740 op_cost(0);
5741 predicate(n->as_Bool()->_test._test == BoolTest::ne
5742 || n->as_Bool()->_test._test == BoolTest::eq);
5743
5744 format %{ "" %}
5745 interface(COND_INTER) %{
5746 equal(0x0, "eq");
5747 not_equal(0x1, "ne");
5748 less(0xb, "lt");
5749 greater_equal(0xa, "ge");
5750 less_equal(0xd, "le");
5751 greater(0xc, "gt");
5752 overflow(0x6, "vs");
5753 no_overflow(0x7, "vc");
5754 %}
5755 %}
5756
5757 // used for certain integral comparisons which can be
5758 // converted to cbxx or tbxx instructions
5759
5760 operand cmpOpLtGe()
5761 %{
5762 match(Bool);
5763 op_cost(0);
5764
5765 predicate(n->as_Bool()->_test._test == BoolTest::lt
5766 || n->as_Bool()->_test._test == BoolTest::ge);
5767
5768 format %{ "" %}
5769 interface(COND_INTER) %{
5770 equal(0x0, "eq");
5771 not_equal(0x1, "ne");
5772 less(0xb, "lt");
5773 greater_equal(0xa, "ge");
5774 less_equal(0xd, "le");
5775 greater(0xc, "gt");
5776 overflow(0x6, "vs");
5777 no_overflow(0x7, "vc");
5778 %}
5779 %}
5780
5781 // used for certain unsigned integral comparisons which can be
5782 // converted to cbxx or tbxx instructions
5783
5784 operand cmpOpUEqNeLeGt()
5785 %{
5786 match(Bool);
5787 op_cost(0);
5788
5789 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5790 n->as_Bool()->_test._test == BoolTest::ne ||
5791 n->as_Bool()->_test._test == BoolTest::le ||
5792 n->as_Bool()->_test._test == BoolTest::gt);
5793
5794 format %{ "" %}
5795 interface(COND_INTER) %{
5796 equal(0x0, "eq");
5797 not_equal(0x1, "ne");
5798 less(0x3, "lo");
5799 greater_equal(0x2, "hs");
5800 less_equal(0x9, "ls");
5801 greater(0x8, "hi");
5802 overflow(0x6, "vs");
5803 no_overflow(0x7, "vc");
5804 %}
5805 %}
5806
5807 // Special operand allowing long args to int ops to be truncated for free
5808
5809 operand iRegL2I(iRegL reg) %{
5810
5811 op_cost(0);
5812
5813 match(ConvL2I reg);
5814
5815 format %{ "l2i($reg)" %}
5816
5817 interface(REG_INTER)
5818 %}
5819
5820 operand iRegL2P(iRegL reg) %{
5821
5822 op_cost(0);
5823
5824 match(CastX2P reg);
5825
5826 format %{ "l2p($reg)" %}
5827
5828 interface(REG_INTER)
5829 %}
5830
5831 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5832 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5833 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5834 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5835
5836 //----------OPERAND CLASSES----------------------------------------------------
5837 // Operand Classes are groups of operands that are used as to simplify
5838 // instruction definitions by not requiring the AD writer to specify
5839 // separate instructions for every form of operand when the
5840 // instruction accepts multiple operand types with the same basic
5841 // encoding and format. The classic case of this is memory operands.
5842
5843 // memory is used to define read/write location for load/store
5844 // instruction defs. we can turn a memory op into an Address
5845
5846 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5847 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5848
5849 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5850 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5851
5852 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5853 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5854
5855 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5856 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5857
5858 // All of the memory operands. For the pipeline description.
5859 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5860 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5861 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5862
5863
5864 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5865 // operations. it allows the src to be either an iRegI or a (ConvL2I
5866 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5867 // can be elided because the 32-bit instruction will just employ the
5868 // lower 32 bits anyway.
5869 //
5870 // n.b. this does not elide all L2I conversions. if the truncated
5871 // value is consumed by more than one operation then the ConvL2I
5872 // cannot be bundled into the consuming nodes so an l2i gets planted
5873 // (actually a movw $dst $src) and the downstream instructions consume
5874 // the result of the l2i as an iRegI input. That's a shame since the
5875 // movw is actually redundant but its not too costly.
5876
5877 opclass iRegIorL2I(iRegI, iRegL2I);
5878 opclass iRegPorL2P(iRegP, iRegL2P);
5879
5880 //----------PIPELINE-----------------------------------------------------------
5881 // Rules which define the behavior of the target architectures pipeline.
5882
5883 // For specific pipelines, eg A53, define the stages of that pipeline
5884 //pipe_desc(ISS, EX1, EX2, WR);
5885 #define ISS S0
5886 #define EX1 S1
5887 #define EX2 S2
5888 #define WR S3
5889
5890 // Integer ALU reg operation
5891 pipeline %{
5892
5893 attributes %{
5894 // ARM instructions are of fixed length
5895 fixed_size_instructions; // Fixed size instructions TODO does
5896 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5897 // ARM instructions come in 32-bit word units
5898 instruction_unit_size = 4; // An instruction is 4 bytes long
5899 instruction_fetch_unit_size = 64; // The processor fetches one line
5900 instruction_fetch_units = 1; // of 64 bytes
5901 %}
5902
5903 // We don't use an actual pipeline model so don't care about resources
5904 // or description. we do use pipeline classes to introduce fixed
5905 // latencies
5906
5907 //----------RESOURCES----------------------------------------------------------
5908 // Resources are the functional units available to the machine
5909
5910 resources( INS0, INS1, INS01 = INS0 | INS1,
5911 ALU0, ALU1, ALU = ALU0 | ALU1,
5912 MAC,
5913 DIV,
5914 BRANCH,
5915 LDST,
5916 NEON_FP);
5917
5918 //----------PIPELINE DESCRIPTION-----------------------------------------------
5919 // Pipeline Description specifies the stages in the machine's pipeline
5920
5921 // Define the pipeline as a generic 6 stage pipeline
5922 pipe_desc(S0, S1, S2, S3, S4, S5);
5923
5924 //----------PIPELINE CLASSES---------------------------------------------------
5925 // Pipeline Classes describe the stages in which input and output are
5926 // referenced by the hardware pipeline.
5927
5928 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5929 %{
5930 single_instruction;
5931 src1 : S1(read);
5932 src2 : S2(read);
5933 dst : S5(write);
5934 INS01 : ISS;
5935 NEON_FP : S5;
5936 %}
5937
5938 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5939 %{
5940 single_instruction;
5941 src1 : S1(read);
5942 src2 : S2(read);
5943 dst : S5(write);
5944 INS01 : ISS;
5945 NEON_FP : S5;
5946 %}
5947
5948 pipe_class fp_uop_s(vRegF dst, vRegF src)
5949 %{
5950 single_instruction;
5951 src : S1(read);
5952 dst : S5(write);
5953 INS01 : ISS;
5954 NEON_FP : S5;
5955 %}
5956
5957 pipe_class fp_uop_d(vRegD dst, vRegD src)
5958 %{
5959 single_instruction;
5960 src : S1(read);
5961 dst : S5(write);
5962 INS01 : ISS;
5963 NEON_FP : S5;
5964 %}
5965
5966 pipe_class fp_d2f(vRegF dst, vRegD src)
5967 %{
5968 single_instruction;
5969 src : S1(read);
5970 dst : S5(write);
5971 INS01 : ISS;
5972 NEON_FP : S5;
5973 %}
5974
5975 pipe_class fp_f2d(vRegD dst, vRegF src)
5976 %{
5977 single_instruction;
5978 src : S1(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5985 %{
5986 single_instruction;
5987 src : S1(read);
5988 dst : S5(write);
5989 INS01 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5994 %{
5995 single_instruction;
5996 src : S1(read);
5997 dst : S5(write);
5998 INS01 : ISS;
5999 NEON_FP : S5;
6000 %}
6001
6002 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6003 %{
6004 single_instruction;
6005 src : S1(read);
6006 dst : S5(write);
6007 INS01 : ISS;
6008 NEON_FP : S5;
6009 %}
6010
6011 pipe_class fp_l2f(vRegF dst, iRegL src)
6012 %{
6013 single_instruction;
6014 src : S1(read);
6015 dst : S5(write);
6016 INS01 : ISS;
6017 NEON_FP : S5;
6018 %}
6019
6020 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6021 %{
6022 single_instruction;
6023 src : S1(read);
6024 dst : S5(write);
6025 INS01 : ISS;
6026 NEON_FP : S5;
6027 %}
6028
6029 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6030 %{
6031 single_instruction;
6032 src : S1(read);
6033 dst : S5(write);
6034 INS01 : ISS;
6035 NEON_FP : S5;
6036 %}
6037
6038 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6039 %{
6040 single_instruction;
6041 src : S1(read);
6042 dst : S5(write);
6043 INS01 : ISS;
6044 NEON_FP : S5;
6045 %}
6046
6047 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6048 %{
6049 single_instruction;
6050 src : S1(read);
6051 dst : S5(write);
6052 INS01 : ISS;
6053 NEON_FP : S5;
6054 %}
6055
6056 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6057 %{
6058 single_instruction;
6059 src1 : S1(read);
6060 src2 : S2(read);
6061 dst : S5(write);
6062 INS0 : ISS;
6063 NEON_FP : S5;
6064 %}
6065
6066 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6067 %{
6068 single_instruction;
6069 src1 : S1(read);
6070 src2 : S2(read);
6071 dst : S5(write);
6072 INS0 : ISS;
6073 NEON_FP : S5;
6074 %}
6075
6076 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6077 %{
6078 single_instruction;
6079 cr : S1(read);
6080 src1 : S1(read);
6081 src2 : S1(read);
6082 dst : S3(write);
6083 INS01 : ISS;
6084 NEON_FP : S3;
6085 %}
6086
6087 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6088 %{
6089 single_instruction;
6090 cr : S1(read);
6091 src1 : S1(read);
6092 src2 : S1(read);
6093 dst : S3(write);
6094 INS01 : ISS;
6095 NEON_FP : S3;
6096 %}
6097
6098 pipe_class fp_imm_s(vRegF dst)
6099 %{
6100 single_instruction;
6101 dst : S3(write);
6102 INS01 : ISS;
6103 NEON_FP : S3;
6104 %}
6105
6106 pipe_class fp_imm_d(vRegD dst)
6107 %{
6108 single_instruction;
6109 dst : S3(write);
6110 INS01 : ISS;
6111 NEON_FP : S3;
6112 %}
6113
6114 pipe_class fp_load_constant_s(vRegF dst)
6115 %{
6116 single_instruction;
6117 dst : S4(write);
6118 INS01 : ISS;
6119 NEON_FP : S4;
6120 %}
6121
6122 pipe_class fp_load_constant_d(vRegD dst)
6123 %{
6124 single_instruction;
6125 dst : S4(write);
6126 INS01 : ISS;
6127 NEON_FP : S4;
6128 %}
6129
6130 //------- Integer ALU operations --------------------------
6131
6132 // Integer ALU reg-reg operation
6133 // Operands needed in EX1, result generated in EX2
6134 // Eg. ADD x0, x1, x2
6135 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6136 %{
6137 single_instruction;
6138 dst : EX2(write);
6139 src1 : EX1(read);
6140 src2 : EX1(read);
6141 INS01 : ISS; // Dual issue as instruction 0 or 1
6142 ALU : EX2;
6143 %}
6144
6145 // Integer ALU reg-reg operation with constant shift
6146 // Shifted register must be available in LATE_ISS instead of EX1
6147 // Eg. ADD x0, x1, x2, LSL #2
6148 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6149 %{
6150 single_instruction;
6151 dst : EX2(write);
6152 src1 : EX1(read);
6153 src2 : ISS(read);
6154 INS01 : ISS;
6155 ALU : EX2;
6156 %}
6157
6158 // Integer ALU reg operation with constant shift
6159 // Eg. LSL x0, x1, #shift
6160 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6161 %{
6162 single_instruction;
6163 dst : EX2(write);
6164 src1 : ISS(read);
6165 INS01 : ISS;
6166 ALU : EX2;
6167 %}
6168
6169 // Integer ALU reg-reg operation with variable shift
6170 // Both operands must be available in LATE_ISS instead of EX1
6171 // Result is available in EX1 instead of EX2
6172 // Eg. LSLV x0, x1, x2
6173 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6174 %{
6175 single_instruction;
6176 dst : EX1(write);
6177 src1 : ISS(read);
6178 src2 : ISS(read);
6179 INS01 : ISS;
6180 ALU : EX1;
6181 %}
6182
6183 // Integer ALU reg-reg operation with extract
6184 // As for _vshift above, but result generated in EX2
6185 // Eg. EXTR x0, x1, x2, #N
6186 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6187 %{
6188 single_instruction;
6189 dst : EX2(write);
6190 src1 : ISS(read);
6191 src2 : ISS(read);
6192 INS1 : ISS; // Can only dual issue as Instruction 1
6193 ALU : EX1;
6194 %}
6195
6196 // Integer ALU reg operation
6197 // Eg. NEG x0, x1
6198 pipe_class ialu_reg(iRegI dst, iRegI src)
6199 %{
6200 single_instruction;
6201 dst : EX2(write);
6202 src : EX1(read);
6203 INS01 : ISS;
6204 ALU : EX2;
6205 %}
6206
6207 // Integer ALU reg mmediate operation
6208 // Eg. ADD x0, x1, #N
6209 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6210 %{
6211 single_instruction;
6212 dst : EX2(write);
6213 src1 : EX1(read);
6214 INS01 : ISS;
6215 ALU : EX2;
6216 %}
6217
6218 // Integer ALU immediate operation (no source operands)
6219 // Eg. MOV x0, #N
6220 pipe_class ialu_imm(iRegI dst)
6221 %{
6222 single_instruction;
6223 dst : EX1(write);
6224 INS01 : ISS;
6225 ALU : EX1;
6226 %}
6227
6228 //------- Compare operation -------------------------------
6229
6230 // Compare reg-reg
6231 // Eg. CMP x0, x1
6232 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6233 %{
6234 single_instruction;
6235 // fixed_latency(16);
6236 cr : EX2(write);
6237 op1 : EX1(read);
6238 op2 : EX1(read);
6239 INS01 : ISS;
6240 ALU : EX2;
6241 %}
6242
6243 // Compare reg-reg
6244 // Eg. CMP x0, #N
6245 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6246 %{
6247 single_instruction;
6248 // fixed_latency(16);
6249 cr : EX2(write);
6250 op1 : EX1(read);
6251 INS01 : ISS;
6252 ALU : EX2;
6253 %}
6254
6255 //------- Conditional instructions ------------------------
6256
6257 // Conditional no operands
6258 // Eg. CSINC x0, zr, zr, <cond>
6259 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6260 %{
6261 single_instruction;
6262 cr : EX1(read);
6263 dst : EX2(write);
6264 INS01 : ISS;
6265 ALU : EX2;
6266 %}
6267
6268 // Conditional 2 operand
6269 // EG. CSEL X0, X1, X2, <cond>
6270 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6271 %{
6272 single_instruction;
6273 cr : EX1(read);
6274 src1 : EX1(read);
6275 src2 : EX1(read);
6276 dst : EX2(write);
6277 INS01 : ISS;
6278 ALU : EX2;
6279 %}
6280
6281 // Conditional 2 operand
6282 // EG. CSEL X0, X1, X2, <cond>
6283 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6284 %{
6285 single_instruction;
6286 cr : EX1(read);
6287 src : EX1(read);
6288 dst : EX2(write);
6289 INS01 : ISS;
6290 ALU : EX2;
6291 %}
6292
6293 //------- Multiply pipeline operations --------------------
6294
6295 // Multiply reg-reg
6296 // Eg. MUL w0, w1, w2
6297 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6298 %{
6299 single_instruction;
6300 dst : WR(write);
6301 src1 : ISS(read);
6302 src2 : ISS(read);
6303 INS01 : ISS;
6304 MAC : WR;
6305 %}
6306
6307 // Multiply accumulate
6308 // Eg. MADD w0, w1, w2, w3
6309 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6310 %{
6311 single_instruction;
6312 dst : WR(write);
6313 src1 : ISS(read);
6314 src2 : ISS(read);
6315 src3 : ISS(read);
6316 INS01 : ISS;
6317 MAC : WR;
6318 %}
6319
6320 // Eg. MUL w0, w1, w2
6321 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6322 %{
6323 single_instruction;
6324 fixed_latency(3); // Maximum latency for 64 bit mul
6325 dst : WR(write);
6326 src1 : ISS(read);
6327 src2 : ISS(read);
6328 INS01 : ISS;
6329 MAC : WR;
6330 %}
6331
6332 // Multiply accumulate
6333 // Eg. MADD w0, w1, w2, w3
6334 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6335 %{
6336 single_instruction;
6337 fixed_latency(3); // Maximum latency for 64 bit mul
6338 dst : WR(write);
6339 src1 : ISS(read);
6340 src2 : ISS(read);
6341 src3 : ISS(read);
6342 INS01 : ISS;
6343 MAC : WR;
6344 %}
6345
6346 //------- Divide pipeline operations --------------------
6347
6348 // Eg. SDIV w0, w1, w2
6349 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6350 %{
6351 single_instruction;
6352 fixed_latency(8); // Maximum latency for 32 bit divide
6353 dst : WR(write);
6354 src1 : ISS(read);
6355 src2 : ISS(read);
6356 INS0 : ISS; // Can only dual issue as instruction 0
6357 DIV : WR;
6358 %}
6359
6360 // Eg. SDIV x0, x1, x2
6361 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6362 %{
6363 single_instruction;
6364 fixed_latency(16); // Maximum latency for 64 bit divide
6365 dst : WR(write);
6366 src1 : ISS(read);
6367 src2 : ISS(read);
6368 INS0 : ISS; // Can only dual issue as instruction 0
6369 DIV : WR;
6370 %}
6371
6372 //------- Load pipeline operations ------------------------
6373
6374 // Load - prefetch
6375 // Eg. PFRM <mem>
6376 pipe_class iload_prefetch(memory mem)
6377 %{
6378 single_instruction;
6379 mem : ISS(read);
6380 INS01 : ISS;
6381 LDST : WR;
6382 %}
6383
6384 // Load - reg, mem
6385 // Eg. LDR x0, <mem>
6386 pipe_class iload_reg_mem(iRegI dst, memory mem)
6387 %{
6388 single_instruction;
6389 dst : WR(write);
6390 mem : ISS(read);
6391 INS01 : ISS;
6392 LDST : WR;
6393 %}
6394
6395 // Load - reg, reg
6396 // Eg. LDR x0, [sp, x1]
6397 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6398 %{
6399 single_instruction;
6400 dst : WR(write);
6401 src : ISS(read);
6402 INS01 : ISS;
6403 LDST : WR;
6404 %}
6405
6406 //------- Store pipeline operations -----------------------
6407
6408 // Store - zr, mem
6409 // Eg. STR zr, <mem>
6410 pipe_class istore_mem(memory mem)
6411 %{
6412 single_instruction;
6413 mem : ISS(read);
6414 INS01 : ISS;
6415 LDST : WR;
6416 %}
6417
6418 // Store - reg, mem
6419 // Eg. STR x0, <mem>
6420 pipe_class istore_reg_mem(iRegI src, memory mem)
6421 %{
6422 single_instruction;
6423 mem : ISS(read);
6424 src : EX2(read);
6425 INS01 : ISS;
6426 LDST : WR;
6427 %}
6428
6429 // Store - reg, reg
6430 // Eg. STR x0, [sp, x1]
6431 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6432 %{
6433 single_instruction;
6434 dst : ISS(read);
6435 src : EX2(read);
6436 INS01 : ISS;
6437 LDST : WR;
6438 %}
6439
6440 //------- Store pipeline operations -----------------------
6441
6442 // Branch
6443 pipe_class pipe_branch()
6444 %{
6445 single_instruction;
6446 INS01 : ISS;
6447 BRANCH : EX1;
6448 %}
6449
6450 // Conditional branch
6451 pipe_class pipe_branch_cond(rFlagsReg cr)
6452 %{
6453 single_instruction;
6454 cr : EX1(read);
6455 INS01 : ISS;
6456 BRANCH : EX1;
6457 %}
6458
6459 // Compare & Branch
6460 // EG. CBZ/CBNZ
6461 pipe_class pipe_cmp_branch(iRegI op1)
6462 %{
6463 single_instruction;
6464 op1 : EX1(read);
6465 INS01 : ISS;
6466 BRANCH : EX1;
6467 %}
6468
6469 //------- Synchronisation operations ----------------------
6470
6471 // Any operation requiring serialization.
6472 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6473 pipe_class pipe_serial()
6474 %{
6475 single_instruction;
6476 force_serialization;
6477 fixed_latency(16);
6478 INS01 : ISS(2); // Cannot dual issue with any other instruction
6479 LDST : WR;
6480 %}
6481
6482 // Generic big/slow expanded idiom - also serialized
6483 pipe_class pipe_slow()
6484 %{
6485 instruction_count(10);
6486 multiple_bundles;
6487 force_serialization;
6488 fixed_latency(16);
6489 INS01 : ISS(2); // Cannot dual issue with any other instruction
6490 LDST : WR;
6491 %}
6492
6493 // Empty pipeline class
6494 pipe_class pipe_class_empty()
6495 %{
6496 single_instruction;
6497 fixed_latency(0);
6498 %}
6499
6500 // Default pipeline class.
6501 pipe_class pipe_class_default()
6502 %{
6503 single_instruction;
6504 fixed_latency(2);
6505 %}
6506
6507 // Pipeline class for compares.
6508 pipe_class pipe_class_compare()
6509 %{
6510 single_instruction;
6511 fixed_latency(16);
6512 %}
6513
6514 // Pipeline class for memory operations.
6515 pipe_class pipe_class_memory()
6516 %{
6517 single_instruction;
6518 fixed_latency(16);
6519 %}
6520
6521 // Pipeline class for call.
6522 pipe_class pipe_class_call()
6523 %{
6524 single_instruction;
6525 fixed_latency(100);
6526 %}
6527
6528 // Define the class for the Nop node.
6529 define %{
6530 MachNop = pipe_class_empty;
6531 %}
6532
6533 %}
6534 //----------INSTRUCTIONS-------------------------------------------------------
6535 //
6536 // match -- States which machine-independent subtree may be replaced
6537 // by this instruction.
6538 // ins_cost -- The estimated cost of this instruction is used by instruction
6539 // selection to identify a minimum cost tree of machine
6540 // instructions that matches a tree of machine-independent
6541 // instructions.
6542 // format -- A string providing the disassembly for this instruction.
6543 // The value of an instruction's operand may be inserted
6544 // by referring to it with a '$' prefix.
6545 // opcode -- Three instruction opcodes may be provided. These are referred
6546 // to within an encode class as $primary, $secondary, and $tertiary
6547 // rrspectively. The primary opcode is commonly used to
6548 // indicate the type of machine instruction, while secondary
6549 // and tertiary are often used for prefix options or addressing
6550 // modes.
6551 // ins_encode -- A list of encode classes with parameters. The encode class
6552 // name must have been defined in an 'enc_class' specification
6553 // in the encode section of the architecture description.
6554
6555 // ============================================================================
6556 // Memory (Load/Store) Instructions
6557
6558 // Load Instructions
6559
6560 // Load Byte (8 bit signed)
6561 instruct loadB(iRegINoSp dst, memory1 mem)
6562 %{
6563 match(Set dst (LoadB mem));
6564 predicate(!needs_acquiring_load(n));
6565
6566 ins_cost(4 * INSN_COST);
6567 format %{ "ldrsbw $dst, $mem\t# byte" %}
6568
6569 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6570
6571 ins_pipe(iload_reg_mem);
6572 %}
6573
6574 // Load Byte (8 bit signed) into long
6575 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6576 %{
6577 match(Set dst (ConvI2L (LoadB mem)));
6578 predicate(!needs_acquiring_load(n->in(1)));
6579
6580 ins_cost(4 * INSN_COST);
6581 format %{ "ldrsb $dst, $mem\t# byte" %}
6582
6583 ins_encode(aarch64_enc_ldrsb(dst, mem));
6584
6585 ins_pipe(iload_reg_mem);
6586 %}
6587
6588 // Load Byte (8 bit unsigned)
6589 instruct loadUB(iRegINoSp dst, memory1 mem)
6590 %{
6591 match(Set dst (LoadUB mem));
6592 predicate(!needs_acquiring_load(n));
6593
6594 ins_cost(4 * INSN_COST);
6595 format %{ "ldrbw $dst, $mem\t# byte" %}
6596
6597 ins_encode(aarch64_enc_ldrb(dst, mem));
6598
6599 ins_pipe(iload_reg_mem);
6600 %}
6601
6602 // Load Byte (8 bit unsigned) into long
6603 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6604 %{
6605 match(Set dst (ConvI2L (LoadUB mem)));
6606 predicate(!needs_acquiring_load(n->in(1)));
6607
6608 ins_cost(4 * INSN_COST);
6609 format %{ "ldrb $dst, $mem\t# byte" %}
6610
6611 ins_encode(aarch64_enc_ldrb(dst, mem));
6612
6613 ins_pipe(iload_reg_mem);
6614 %}
6615
6616 // Load Short (16 bit signed)
6617 instruct loadS(iRegINoSp dst, memory2 mem)
6618 %{
6619 match(Set dst (LoadS mem));
6620 predicate(!needs_acquiring_load(n));
6621
6622 ins_cost(4 * INSN_COST);
6623 format %{ "ldrshw $dst, $mem\t# short" %}
6624
6625 ins_encode(aarch64_enc_ldrshw(dst, mem));
6626
6627 ins_pipe(iload_reg_mem);
6628 %}
6629
6630 // Load Short (16 bit signed) into long
6631 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6632 %{
6633 match(Set dst (ConvI2L (LoadS mem)));
6634 predicate(!needs_acquiring_load(n->in(1)));
6635
6636 ins_cost(4 * INSN_COST);
6637 format %{ "ldrsh $dst, $mem\t# short" %}
6638
6639 ins_encode(aarch64_enc_ldrsh(dst, mem));
6640
6641 ins_pipe(iload_reg_mem);
6642 %}
6643
6644 // Load Char (16 bit unsigned)
6645 instruct loadUS(iRegINoSp dst, memory2 mem)
6646 %{
6647 match(Set dst (LoadUS mem));
6648 predicate(!needs_acquiring_load(n));
6649
6650 ins_cost(4 * INSN_COST);
6651 format %{ "ldrh $dst, $mem\t# short" %}
6652
6653 ins_encode(aarch64_enc_ldrh(dst, mem));
6654
6655 ins_pipe(iload_reg_mem);
6656 %}
6657
6658 // Load Short/Char (16 bit unsigned) into long
6659 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6660 %{
6661 match(Set dst (ConvI2L (LoadUS mem)));
6662 predicate(!needs_acquiring_load(n->in(1)));
6663
6664 ins_cost(4 * INSN_COST);
6665 format %{ "ldrh $dst, $mem\t# short" %}
6666
6667 ins_encode(aarch64_enc_ldrh(dst, mem));
6668
6669 ins_pipe(iload_reg_mem);
6670 %}
6671
6672 // Load Integer (32 bit signed)
6673 instruct loadI(iRegINoSp dst, memory4 mem)
6674 %{
6675 match(Set dst (LoadI mem));
6676 predicate(!needs_acquiring_load(n));
6677
6678 ins_cost(4 * INSN_COST);
6679 format %{ "ldrw $dst, $mem\t# int" %}
6680
6681 ins_encode(aarch64_enc_ldrw(dst, mem));
6682
6683 ins_pipe(iload_reg_mem);
6684 %}
6685
6686 // Load Integer (32 bit signed) into long
6687 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6688 %{
6689 match(Set dst (ConvI2L (LoadI mem)));
6690 predicate(!needs_acquiring_load(n->in(1)));
6691
6692 ins_cost(4 * INSN_COST);
6693 format %{ "ldrsw $dst, $mem\t# int" %}
6694
6695 ins_encode(aarch64_enc_ldrsw(dst, mem));
6696
6697 ins_pipe(iload_reg_mem);
6698 %}
6699
6700 // Load Integer (32 bit unsigned) into long
6701 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6702 %{
6703 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6704 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6705
6706 ins_cost(4 * INSN_COST);
6707 format %{ "ldrw $dst, $mem\t# int" %}
6708
6709 ins_encode(aarch64_enc_ldrw(dst, mem));
6710
6711 ins_pipe(iload_reg_mem);
6712 %}
6713
6714 // Load Long (64 bit signed)
6715 instruct loadL(iRegLNoSp dst, memory8 mem)
6716 %{
6717 match(Set dst (LoadL mem));
6718 predicate(!needs_acquiring_load(n));
6719
6720 ins_cost(4 * INSN_COST);
6721 format %{ "ldr $dst, $mem\t# int" %}
6722
6723 ins_encode(aarch64_enc_ldr(dst, mem));
6724
6725 ins_pipe(iload_reg_mem);
6726 %}
6727
6728 // Load Range
6729 instruct loadRange(iRegINoSp dst, memory4 mem)
6730 %{
6731 match(Set dst (LoadRange mem));
6732
6733 ins_cost(4 * INSN_COST);
6734 format %{ "ldrw $dst, $mem\t# range" %}
6735
6736 ins_encode(aarch64_enc_ldrw(dst, mem));
6737
6738 ins_pipe(iload_reg_mem);
6739 %}
6740
6741 // Load Pointer
6742 instruct loadP(iRegPNoSp dst, memory8 mem)
6743 %{
6744 match(Set dst (LoadP mem));
6745 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6746
6747 ins_cost(4 * INSN_COST);
6748 format %{ "ldr $dst, $mem\t# ptr" %}
6749
6750 ins_encode(aarch64_enc_ldr(dst, mem));
6751
6752 ins_pipe(iload_reg_mem);
6753 %}
6754
6755 // Load Compressed Pointer
6756 instruct loadN(iRegNNoSp dst, memory4 mem)
6757 %{
6758 match(Set dst (LoadN mem));
6759 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6760
6761 ins_cost(4 * INSN_COST);
6762 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6763
6764 ins_encode(aarch64_enc_ldrw(dst, mem));
6765
6766 ins_pipe(iload_reg_mem);
6767 %}
6768
6769 // Load Klass Pointer
6770 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6771 %{
6772 match(Set dst (LoadKlass mem));
6773 predicate(!needs_acquiring_load(n));
6774
6775 ins_cost(4 * INSN_COST);
6776 format %{ "ldr $dst, $mem\t# class" %}
6777
6778 ins_encode(aarch64_enc_ldr(dst, mem));
6779
6780 ins_pipe(iload_reg_mem);
6781 %}
6782
6783 // Load Narrow Klass Pointer
6784 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6785 %{
6786 match(Set dst (LoadNKlass mem));
6787 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6788
6789 ins_cost(4 * INSN_COST);
6790 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6791
6792 ins_encode(aarch64_enc_ldrw(dst, mem));
6793
6794 ins_pipe(iload_reg_mem);
6795 %}
6796
6797 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6798 %{
6799 match(Set dst (LoadNKlass mem));
6800 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6801
6802 ins_cost(4 * INSN_COST);
6803 format %{
6804 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6805 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6806 %}
6807 ins_encode %{
6808 // inlined aarch64_enc_ldrw
6809 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6810 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6811 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6812 %}
6813 ins_pipe(iload_reg_mem);
6814 %}
6815
6816 // Load Float
6817 instruct loadF(vRegF dst, memory4 mem)
6818 %{
6819 match(Set dst (LoadF mem));
6820 predicate(!needs_acquiring_load(n));
6821
6822 ins_cost(4 * INSN_COST);
6823 format %{ "ldrs $dst, $mem\t# float" %}
6824
6825 ins_encode( aarch64_enc_ldrs(dst, mem) );
6826
6827 ins_pipe(pipe_class_memory);
6828 %}
6829
6830 // Load Double
6831 instruct loadD(vRegD dst, memory8 mem)
6832 %{
6833 match(Set dst (LoadD mem));
6834 predicate(!needs_acquiring_load(n));
6835
6836 ins_cost(4 * INSN_COST);
6837 format %{ "ldrd $dst, $mem\t# double" %}
6838
6839 ins_encode( aarch64_enc_ldrd(dst, mem) );
6840
6841 ins_pipe(pipe_class_memory);
6842 %}
6843
6844
6845 // Load Int Constant
6846 instruct loadConI(iRegINoSp dst, immI src)
6847 %{
6848 match(Set dst src);
6849
6850 ins_cost(INSN_COST);
6851 format %{ "mov $dst, $src\t# int" %}
6852
6853 ins_encode( aarch64_enc_movw_imm(dst, src) );
6854
6855 ins_pipe(ialu_imm);
6856 %}
6857
6858 // Load Long Constant
6859 instruct loadConL(iRegLNoSp dst, immL src)
6860 %{
6861 match(Set dst src);
6862
6863 ins_cost(INSN_COST);
6864 format %{ "mov $dst, $src\t# long" %}
6865
6866 ins_encode( aarch64_enc_mov_imm(dst, src) );
6867
6868 ins_pipe(ialu_imm);
6869 %}
6870
6871 // Load Pointer Constant
6872
6873 instruct loadConP(iRegPNoSp dst, immP con)
6874 %{
6875 match(Set dst con);
6876
6877 ins_cost(INSN_COST * 4);
6878 format %{
6879 "mov $dst, $con\t# ptr\n\t"
6880 %}
6881
6882 ins_encode(aarch64_enc_mov_p(dst, con));
6883
6884 ins_pipe(ialu_imm);
6885 %}
6886
6887 // Load Null Pointer Constant
6888
6889 instruct loadConP0(iRegPNoSp dst, immP0 con)
6890 %{
6891 match(Set dst con);
6892
6893 ins_cost(INSN_COST);
6894 format %{ "mov $dst, $con\t# nullptr ptr" %}
6895
6896 ins_encode(aarch64_enc_mov_p0(dst, con));
6897
6898 ins_pipe(ialu_imm);
6899 %}
6900
6901 // Load Pointer Constant One
6902
6903 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6904 %{
6905 match(Set dst con);
6906
6907 ins_cost(INSN_COST);
6908 format %{ "mov $dst, $con\t# nullptr ptr" %}
6909
6910 ins_encode(aarch64_enc_mov_p1(dst, con));
6911
6912 ins_pipe(ialu_imm);
6913 %}
6914
6915 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6916 %{
6917 match(Set dst con);
6918
6919 ins_cost(INSN_COST);
6920 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6921
6922 ins_encode %{
6923 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6924 %}
6925
6926 ins_pipe(ialu_imm);
6927 %}
6928
6929 // Load Narrow Pointer Constant
6930
6931 instruct loadConN(iRegNNoSp dst, immN con)
6932 %{
6933 match(Set dst con);
6934
6935 ins_cost(INSN_COST * 4);
6936 format %{ "mov $dst, $con\t# compressed ptr" %}
6937
6938 ins_encode(aarch64_enc_mov_n(dst, con));
6939
6940 ins_pipe(ialu_imm);
6941 %}
6942
6943 // Load Narrow Null Pointer Constant
6944
6945 instruct loadConN0(iRegNNoSp dst, immN0 con)
6946 %{
6947 match(Set dst con);
6948
6949 ins_cost(INSN_COST);
6950 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6951
6952 ins_encode(aarch64_enc_mov_n0(dst, con));
6953
6954 ins_pipe(ialu_imm);
6955 %}
6956
6957 // Load Narrow Klass Constant
6958
6959 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6960 %{
6961 match(Set dst con);
6962
6963 ins_cost(INSN_COST);
6964 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6965
6966 ins_encode(aarch64_enc_mov_nk(dst, con));
6967
6968 ins_pipe(ialu_imm);
6969 %}
6970
6971 // Load Packed Float Constant
6972
6973 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6974 match(Set dst con);
6975 ins_cost(INSN_COST * 4);
6976 format %{ "fmovs $dst, $con"%}
6977 ins_encode %{
6978 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6979 %}
6980
6981 ins_pipe(fp_imm_s);
6982 %}
6983
6984 // Load Float Constant
6985
6986 instruct loadConF(vRegF dst, immF con) %{
6987 match(Set dst con);
6988
6989 ins_cost(INSN_COST * 4);
6990
6991 format %{
6992 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6993 %}
6994
6995 ins_encode %{
6996 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6997 %}
6998
6999 ins_pipe(fp_load_constant_s);
7000 %}
7001
7002 // Load Packed Double Constant
7003
7004 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7005 match(Set dst con);
7006 ins_cost(INSN_COST);
7007 format %{ "fmovd $dst, $con"%}
7008 ins_encode %{
7009 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7010 %}
7011
7012 ins_pipe(fp_imm_d);
7013 %}
7014
7015 // Load Double Constant
7016
7017 instruct loadConD(vRegD dst, immD con) %{
7018 match(Set dst con);
7019
7020 ins_cost(INSN_COST * 5);
7021 format %{
7022 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7023 %}
7024
7025 ins_encode %{
7026 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7027 %}
7028
7029 ins_pipe(fp_load_constant_d);
7030 %}
7031
7032 // Load Half Float Constant
7033 instruct loadConH(vRegF dst, immH con) %{
7034 match(Set dst con);
7035 format %{ "mov rscratch1, $con\n\t"
7036 "fmov $dst, rscratch1"
7037 %}
7038 ins_encode %{
7039 __ movw(rscratch1, (uint32_t)$con$$constant);
7040 __ fmovs($dst$$FloatRegister, rscratch1);
7041 %}
7042 ins_pipe(pipe_class_default);
7043 %}
7044
7045 // Store Instructions
7046
7047 // Store Byte
7048 instruct storeB(iRegIorL2I src, memory1 mem)
7049 %{
7050 match(Set mem (StoreB mem src));
7051 predicate(!needs_releasing_store(n));
7052
7053 ins_cost(INSN_COST);
7054 format %{ "strb $src, $mem\t# byte" %}
7055
7056 ins_encode(aarch64_enc_strb(src, mem));
7057
7058 ins_pipe(istore_reg_mem);
7059 %}
7060
7061
7062 instruct storeimmB0(immI0 zero, memory1 mem)
7063 %{
7064 match(Set mem (StoreB mem zero));
7065 predicate(!needs_releasing_store(n));
7066
7067 ins_cost(INSN_COST);
7068 format %{ "strb rscractch2, $mem\t# byte" %}
7069
7070 ins_encode(aarch64_enc_strb0(mem));
7071
7072 ins_pipe(istore_mem);
7073 %}
7074
7075 // Store Char/Short
7076 instruct storeC(iRegIorL2I src, memory2 mem)
7077 %{
7078 match(Set mem (StoreC mem src));
7079 predicate(!needs_releasing_store(n));
7080
7081 ins_cost(INSN_COST);
7082 format %{ "strh $src, $mem\t# short" %}
7083
7084 ins_encode(aarch64_enc_strh(src, mem));
7085
7086 ins_pipe(istore_reg_mem);
7087 %}
7088
7089 instruct storeimmC0(immI0 zero, memory2 mem)
7090 %{
7091 match(Set mem (StoreC mem zero));
7092 predicate(!needs_releasing_store(n));
7093
7094 ins_cost(INSN_COST);
7095 format %{ "strh zr, $mem\t# short" %}
7096
7097 ins_encode(aarch64_enc_strh0(mem));
7098
7099 ins_pipe(istore_mem);
7100 %}
7101
7102 // Store Integer
7103
7104 instruct storeI(iRegIorL2I src, memory4 mem)
7105 %{
7106 match(Set mem(StoreI mem src));
7107 predicate(!needs_releasing_store(n));
7108
7109 ins_cost(INSN_COST);
7110 format %{ "strw $src, $mem\t# int" %}
7111
7112 ins_encode(aarch64_enc_strw(src, mem));
7113
7114 ins_pipe(istore_reg_mem);
7115 %}
7116
7117 instruct storeimmI0(immI0 zero, memory4 mem)
7118 %{
7119 match(Set mem(StoreI mem zero));
7120 predicate(!needs_releasing_store(n));
7121
7122 ins_cost(INSN_COST);
7123 format %{ "strw zr, $mem\t# int" %}
7124
7125 ins_encode(aarch64_enc_strw0(mem));
7126
7127 ins_pipe(istore_mem);
7128 %}
7129
7130 // Store Long (64 bit signed)
7131 instruct storeL(iRegL src, memory8 mem)
7132 %{
7133 match(Set mem (StoreL mem src));
7134 predicate(!needs_releasing_store(n));
7135
7136 ins_cost(INSN_COST);
7137 format %{ "str $src, $mem\t# int" %}
7138
7139 ins_encode(aarch64_enc_str(src, mem));
7140
7141 ins_pipe(istore_reg_mem);
7142 %}
7143
7144 // Store Long (64 bit signed)
7145 instruct storeimmL0(immL0 zero, memory8 mem)
7146 %{
7147 match(Set mem (StoreL mem zero));
7148 predicate(!needs_releasing_store(n));
7149
7150 ins_cost(INSN_COST);
7151 format %{ "str zr, $mem\t# int" %}
7152
7153 ins_encode(aarch64_enc_str0(mem));
7154
7155 ins_pipe(istore_mem);
7156 %}
7157
7158 // Store Pointer
7159 instruct storeP(iRegP src, memory8 mem)
7160 %{
7161 match(Set mem (StoreP mem src));
7162 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7163
7164 ins_cost(INSN_COST);
7165 format %{ "str $src, $mem\t# ptr" %}
7166
7167 ins_encode(aarch64_enc_str(src, mem));
7168
7169 ins_pipe(istore_reg_mem);
7170 %}
7171
7172 // Store Pointer
7173 instruct storeimmP0(immP0 zero, memory8 mem)
7174 %{
7175 match(Set mem (StoreP mem zero));
7176 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7177
7178 ins_cost(INSN_COST);
7179 format %{ "str zr, $mem\t# ptr" %}
7180
7181 ins_encode(aarch64_enc_str0(mem));
7182
7183 ins_pipe(istore_mem);
7184 %}
7185
7186 // Store Compressed Pointer
7187 instruct storeN(iRegN src, memory4 mem)
7188 %{
7189 match(Set mem (StoreN mem src));
7190 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7191
7192 ins_cost(INSN_COST);
7193 format %{ "strw $src, $mem\t# compressed ptr" %}
7194
7195 ins_encode(aarch64_enc_strw(src, mem));
7196
7197 ins_pipe(istore_reg_mem);
7198 %}
7199
7200 instruct storeImmN0(immN0 zero, memory4 mem)
7201 %{
7202 match(Set mem (StoreN mem zero));
7203 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7204
7205 ins_cost(INSN_COST);
7206 format %{ "strw zr, $mem\t# compressed ptr" %}
7207
7208 ins_encode(aarch64_enc_strw0(mem));
7209
7210 ins_pipe(istore_mem);
7211 %}
7212
7213 // Store Float
7214 instruct storeF(vRegF src, memory4 mem)
7215 %{
7216 match(Set mem (StoreF mem src));
7217 predicate(!needs_releasing_store(n));
7218
7219 ins_cost(INSN_COST);
7220 format %{ "strs $src, $mem\t# float" %}
7221
7222 ins_encode( aarch64_enc_strs(src, mem) );
7223
7224 ins_pipe(pipe_class_memory);
7225 %}
7226
7227 // TODO
7228 // implement storeImmF0 and storeFImmPacked
7229
7230 // Store Double
7231 instruct storeD(vRegD src, memory8 mem)
7232 %{
7233 match(Set mem (StoreD mem src));
7234 predicate(!needs_releasing_store(n));
7235
7236 ins_cost(INSN_COST);
7237 format %{ "strd $src, $mem\t# double" %}
7238
7239 ins_encode( aarch64_enc_strd(src, mem) );
7240
7241 ins_pipe(pipe_class_memory);
7242 %}
7243
7244 // Store Compressed Klass Pointer
7245 instruct storeNKlass(iRegN src, memory4 mem)
7246 %{
7247 predicate(!needs_releasing_store(n));
7248 match(Set mem (StoreNKlass mem src));
7249
7250 ins_cost(INSN_COST);
7251 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7252
7253 ins_encode(aarch64_enc_strw(src, mem));
7254
7255 ins_pipe(istore_reg_mem);
7256 %}
7257
7258 // TODO
7259 // implement storeImmD0 and storeDImmPacked
7260
7261 // prefetch instructions
7262 // Must be safe to execute with invalid address (cannot fault).
7263
7264 instruct prefetchalloc( memory8 mem ) %{
7265 match(PrefetchAllocation mem);
7266
7267 ins_cost(INSN_COST);
7268 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7269
7270 ins_encode( aarch64_enc_prefetchw(mem) );
7271
7272 ins_pipe(iload_prefetch);
7273 %}
7274
7275 // ---------------- volatile loads and stores ----------------
7276
7277 // Load Byte (8 bit signed)
7278 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7279 %{
7280 match(Set dst (LoadB mem));
7281
7282 ins_cost(VOLATILE_REF_COST);
7283 format %{ "ldarsb $dst, $mem\t# byte" %}
7284
7285 ins_encode(aarch64_enc_ldarsb(dst, mem));
7286
7287 ins_pipe(pipe_serial);
7288 %}
7289
7290 // Load Byte (8 bit signed) into long
7291 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7292 %{
7293 match(Set dst (ConvI2L (LoadB mem)));
7294
7295 ins_cost(VOLATILE_REF_COST);
7296 format %{ "ldarsb $dst, $mem\t# byte" %}
7297
7298 ins_encode(aarch64_enc_ldarsb(dst, mem));
7299
7300 ins_pipe(pipe_serial);
7301 %}
7302
7303 // Load Byte (8 bit unsigned)
7304 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7305 %{
7306 match(Set dst (LoadUB mem));
7307
7308 ins_cost(VOLATILE_REF_COST);
7309 format %{ "ldarb $dst, $mem\t# byte" %}
7310
7311 ins_encode(aarch64_enc_ldarb(dst, mem));
7312
7313 ins_pipe(pipe_serial);
7314 %}
7315
7316 // Load Byte (8 bit unsigned) into long
7317 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7318 %{
7319 match(Set dst (ConvI2L (LoadUB mem)));
7320
7321 ins_cost(VOLATILE_REF_COST);
7322 format %{ "ldarb $dst, $mem\t# byte" %}
7323
7324 ins_encode(aarch64_enc_ldarb(dst, mem));
7325
7326 ins_pipe(pipe_serial);
7327 %}
7328
7329 // Load Short (16 bit signed)
7330 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7331 %{
7332 match(Set dst (LoadS mem));
7333
7334 ins_cost(VOLATILE_REF_COST);
7335 format %{ "ldarshw $dst, $mem\t# short" %}
7336
7337 ins_encode(aarch64_enc_ldarshw(dst, mem));
7338
7339 ins_pipe(pipe_serial);
7340 %}
7341
7342 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7343 %{
7344 match(Set dst (LoadUS mem));
7345
7346 ins_cost(VOLATILE_REF_COST);
7347 format %{ "ldarhw $dst, $mem\t# short" %}
7348
7349 ins_encode(aarch64_enc_ldarhw(dst, mem));
7350
7351 ins_pipe(pipe_serial);
7352 %}
7353
7354 // Load Short/Char (16 bit unsigned) into long
7355 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7356 %{
7357 match(Set dst (ConvI2L (LoadUS mem)));
7358
7359 ins_cost(VOLATILE_REF_COST);
7360 format %{ "ldarh $dst, $mem\t# short" %}
7361
7362 ins_encode(aarch64_enc_ldarh(dst, mem));
7363
7364 ins_pipe(pipe_serial);
7365 %}
7366
7367 // Load Short/Char (16 bit signed) into long
7368 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7369 %{
7370 match(Set dst (ConvI2L (LoadS mem)));
7371
7372 ins_cost(VOLATILE_REF_COST);
7373 format %{ "ldarh $dst, $mem\t# short" %}
7374
7375 ins_encode(aarch64_enc_ldarsh(dst, mem));
7376
7377 ins_pipe(pipe_serial);
7378 %}
7379
7380 // Load Integer (32 bit signed)
7381 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7382 %{
7383 match(Set dst (LoadI mem));
7384
7385 ins_cost(VOLATILE_REF_COST);
7386 format %{ "ldarw $dst, $mem\t# int" %}
7387
7388 ins_encode(aarch64_enc_ldarw(dst, mem));
7389
7390 ins_pipe(pipe_serial);
7391 %}
7392
7393 // Load Integer (32 bit unsigned) into long
7394 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7395 %{
7396 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7397
7398 ins_cost(VOLATILE_REF_COST);
7399 format %{ "ldarw $dst, $mem\t# int" %}
7400
7401 ins_encode(aarch64_enc_ldarw(dst, mem));
7402
7403 ins_pipe(pipe_serial);
7404 %}
7405
7406 // Load Long (64 bit signed)
7407 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7408 %{
7409 match(Set dst (LoadL mem));
7410
7411 ins_cost(VOLATILE_REF_COST);
7412 format %{ "ldar $dst, $mem\t# int" %}
7413
7414 ins_encode(aarch64_enc_ldar(dst, mem));
7415
7416 ins_pipe(pipe_serial);
7417 %}
7418
7419 // Load Pointer
7420 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7421 %{
7422 match(Set dst (LoadP mem));
7423 predicate(n->as_Load()->barrier_data() == 0);
7424
7425 ins_cost(VOLATILE_REF_COST);
7426 format %{ "ldar $dst, $mem\t# ptr" %}
7427
7428 ins_encode(aarch64_enc_ldar(dst, mem));
7429
7430 ins_pipe(pipe_serial);
7431 %}
7432
7433 // Load Compressed Pointer
7434 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7435 %{
7436 match(Set dst (LoadN mem));
7437 predicate(n->as_Load()->barrier_data() == 0);
7438
7439 ins_cost(VOLATILE_REF_COST);
7440 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7441
7442 ins_encode(aarch64_enc_ldarw(dst, mem));
7443
7444 ins_pipe(pipe_serial);
7445 %}
7446
7447 // Load Float
7448 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7449 %{
7450 match(Set dst (LoadF mem));
7451
7452 ins_cost(VOLATILE_REF_COST);
7453 format %{ "ldars $dst, $mem\t# float" %}
7454
7455 ins_encode( aarch64_enc_fldars(dst, mem) );
7456
7457 ins_pipe(pipe_serial);
7458 %}
7459
7460 // Load Double
7461 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7462 %{
7463 match(Set dst (LoadD mem));
7464
7465 ins_cost(VOLATILE_REF_COST);
7466 format %{ "ldard $dst, $mem\t# double" %}
7467
7468 ins_encode( aarch64_enc_fldard(dst, mem) );
7469
7470 ins_pipe(pipe_serial);
7471 %}
7472
7473 // Store Byte
7474 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7475 %{
7476 match(Set mem (StoreB mem src));
7477
7478 ins_cost(VOLATILE_REF_COST);
7479 format %{ "stlrb $src, $mem\t# byte" %}
7480
7481 ins_encode(aarch64_enc_stlrb(src, mem));
7482
7483 ins_pipe(pipe_class_memory);
7484 %}
7485
7486 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7487 %{
7488 match(Set mem (StoreB mem zero));
7489
7490 ins_cost(VOLATILE_REF_COST);
7491 format %{ "stlrb zr, $mem\t# byte" %}
7492
7493 ins_encode(aarch64_enc_stlrb0(mem));
7494
7495 ins_pipe(pipe_class_memory);
7496 %}
7497
7498 // Store Char/Short
7499 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7500 %{
7501 match(Set mem (StoreC mem src));
7502
7503 ins_cost(VOLATILE_REF_COST);
7504 format %{ "stlrh $src, $mem\t# short" %}
7505
7506 ins_encode(aarch64_enc_stlrh(src, mem));
7507
7508 ins_pipe(pipe_class_memory);
7509 %}
7510
7511 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7512 %{
7513 match(Set mem (StoreC mem zero));
7514
7515 ins_cost(VOLATILE_REF_COST);
7516 format %{ "stlrh zr, $mem\t# short" %}
7517
7518 ins_encode(aarch64_enc_stlrh0(mem));
7519
7520 ins_pipe(pipe_class_memory);
7521 %}
7522
7523 // Store Integer
7524
7525 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7526 %{
7527 match(Set mem(StoreI mem src));
7528
7529 ins_cost(VOLATILE_REF_COST);
7530 format %{ "stlrw $src, $mem\t# int" %}
7531
7532 ins_encode(aarch64_enc_stlrw(src, mem));
7533
7534 ins_pipe(pipe_class_memory);
7535 %}
7536
7537 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7538 %{
7539 match(Set mem(StoreI mem zero));
7540
7541 ins_cost(VOLATILE_REF_COST);
7542 format %{ "stlrw zr, $mem\t# int" %}
7543
7544 ins_encode(aarch64_enc_stlrw0(mem));
7545
7546 ins_pipe(pipe_class_memory);
7547 %}
7548
7549 // Store Long (64 bit signed)
7550 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7551 %{
7552 match(Set mem (StoreL mem src));
7553
7554 ins_cost(VOLATILE_REF_COST);
7555 format %{ "stlr $src, $mem\t# int" %}
7556
7557 ins_encode(aarch64_enc_stlr(src, mem));
7558
7559 ins_pipe(pipe_class_memory);
7560 %}
7561
7562 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7563 %{
7564 match(Set mem (StoreL mem zero));
7565
7566 ins_cost(VOLATILE_REF_COST);
7567 format %{ "stlr zr, $mem\t# int" %}
7568
7569 ins_encode(aarch64_enc_stlr0(mem));
7570
7571 ins_pipe(pipe_class_memory);
7572 %}
7573
7574 // Store Pointer
7575 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7576 %{
7577 match(Set mem (StoreP mem src));
7578 predicate(n->as_Store()->barrier_data() == 0);
7579
7580 ins_cost(VOLATILE_REF_COST);
7581 format %{ "stlr $src, $mem\t# ptr" %}
7582
7583 ins_encode(aarch64_enc_stlr(src, mem));
7584
7585 ins_pipe(pipe_class_memory);
7586 %}
7587
7588 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7589 %{
7590 match(Set mem (StoreP mem zero));
7591 predicate(n->as_Store()->barrier_data() == 0);
7592
7593 ins_cost(VOLATILE_REF_COST);
7594 format %{ "stlr zr, $mem\t# ptr" %}
7595
7596 ins_encode(aarch64_enc_stlr0(mem));
7597
7598 ins_pipe(pipe_class_memory);
7599 %}
7600
7601 // Store Compressed Pointer
7602 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7603 %{
7604 match(Set mem (StoreN mem src));
7605 predicate(n->as_Store()->barrier_data() == 0);
7606
7607 ins_cost(VOLATILE_REF_COST);
7608 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7609
7610 ins_encode(aarch64_enc_stlrw(src, mem));
7611
7612 ins_pipe(pipe_class_memory);
7613 %}
7614
7615 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7616 %{
7617 match(Set mem (StoreN mem zero));
7618 predicate(n->as_Store()->barrier_data() == 0);
7619
7620 ins_cost(VOLATILE_REF_COST);
7621 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7622
7623 ins_encode(aarch64_enc_stlrw0(mem));
7624
7625 ins_pipe(pipe_class_memory);
7626 %}
7627
7628 // Store Float
7629 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7630 %{
7631 match(Set mem (StoreF mem src));
7632
7633 ins_cost(VOLATILE_REF_COST);
7634 format %{ "stlrs $src, $mem\t# float" %}
7635
7636 ins_encode( aarch64_enc_fstlrs(src, mem) );
7637
7638 ins_pipe(pipe_class_memory);
7639 %}
7640
7641 // TODO
7642 // implement storeImmF0 and storeFImmPacked
7643
7644 // Store Double
7645 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7646 %{
7647 match(Set mem (StoreD mem src));
7648
7649 ins_cost(VOLATILE_REF_COST);
7650 format %{ "stlrd $src, $mem\t# double" %}
7651
7652 ins_encode( aarch64_enc_fstlrd(src, mem) );
7653
7654 ins_pipe(pipe_class_memory);
7655 %}
7656
7657 // ---------------- end of volatile loads and stores ----------------
7658
7659 instruct cacheWB(indirect addr)
7660 %{
7661 predicate(VM_Version::supports_data_cache_line_flush());
7662 match(CacheWB addr);
7663
7664 ins_cost(100);
7665 format %{"cache wb $addr" %}
7666 ins_encode %{
7667 assert($addr->index_position() < 0, "should be");
7668 assert($addr$$disp == 0, "should be");
7669 __ cache_wb(Address($addr$$base$$Register, 0));
7670 %}
7671 ins_pipe(pipe_slow); // XXX
7672 %}
7673
7674 instruct cacheWBPreSync()
7675 %{
7676 predicate(VM_Version::supports_data_cache_line_flush());
7677 match(CacheWBPreSync);
7678
7679 ins_cost(100);
7680 format %{"cache wb presync" %}
7681 ins_encode %{
7682 __ cache_wbsync(true);
7683 %}
7684 ins_pipe(pipe_slow); // XXX
7685 %}
7686
7687 instruct cacheWBPostSync()
7688 %{
7689 predicate(VM_Version::supports_data_cache_line_flush());
7690 match(CacheWBPostSync);
7691
7692 ins_cost(100);
7693 format %{"cache wb postsync" %}
7694 ins_encode %{
7695 __ cache_wbsync(false);
7696 %}
7697 ins_pipe(pipe_slow); // XXX
7698 %}
7699
7700 // ============================================================================
7701 // BSWAP Instructions
7702
7703 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7704 match(Set dst (ReverseBytesI src));
7705
7706 ins_cost(INSN_COST);
7707 format %{ "revw $dst, $src" %}
7708
7709 ins_encode %{
7710 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7711 %}
7712
7713 ins_pipe(ialu_reg);
7714 %}
7715
7716 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7717 match(Set dst (ReverseBytesL src));
7718
7719 ins_cost(INSN_COST);
7720 format %{ "rev $dst, $src" %}
7721
7722 ins_encode %{
7723 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7724 %}
7725
7726 ins_pipe(ialu_reg);
7727 %}
7728
7729 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7730 match(Set dst (ReverseBytesUS src));
7731
7732 ins_cost(INSN_COST);
7733 format %{ "rev16w $dst, $src" %}
7734
7735 ins_encode %{
7736 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7737 %}
7738
7739 ins_pipe(ialu_reg);
7740 %}
7741
7742 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7743 match(Set dst (ReverseBytesS src));
7744
7745 ins_cost(INSN_COST);
7746 format %{ "rev16w $dst, $src\n\t"
7747 "sbfmw $dst, $dst, #0, #15" %}
7748
7749 ins_encode %{
7750 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7751 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7752 %}
7753
7754 ins_pipe(ialu_reg);
7755 %}
7756
7757 // ============================================================================
7758 // Zero Count Instructions
7759
7760 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7761 match(Set dst (CountLeadingZerosI src));
7762
7763 ins_cost(INSN_COST);
7764 format %{ "clzw $dst, $src" %}
7765 ins_encode %{
7766 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7767 %}
7768
7769 ins_pipe(ialu_reg);
7770 %}
7771
7772 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7773 match(Set dst (CountLeadingZerosL src));
7774
7775 ins_cost(INSN_COST);
7776 format %{ "clz $dst, $src" %}
7777 ins_encode %{
7778 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7779 %}
7780
7781 ins_pipe(ialu_reg);
7782 %}
7783
7784 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7785 match(Set dst (CountTrailingZerosI src));
7786
7787 ins_cost(INSN_COST * 2);
7788 format %{ "rbitw $dst, $src\n\t"
7789 "clzw $dst, $dst" %}
7790 ins_encode %{
7791 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7792 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7793 %}
7794
7795 ins_pipe(ialu_reg);
7796 %}
7797
7798 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7799 match(Set dst (CountTrailingZerosL src));
7800
7801 ins_cost(INSN_COST * 2);
7802 format %{ "rbit $dst, $src\n\t"
7803 "clz $dst, $dst" %}
7804 ins_encode %{
7805 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7806 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7807 %}
7808
7809 ins_pipe(ialu_reg);
7810 %}
7811
7812 //---------- Population Count Instructions -------------------------------------
7813 //
7814
7815 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7816 match(Set dst (PopCountI src));
7817 effect(TEMP tmp);
7818 ins_cost(INSN_COST * 13);
7819
7820 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7821 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7822 "addv $tmp, $tmp\t# vector (8B)\n\t"
7823 "mov $dst, $tmp\t# vector (1D)" %}
7824 ins_encode %{
7825 __ fmovs($tmp$$FloatRegister, $src$$Register);
7826 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7827 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7828 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7829 %}
7830
7831 ins_pipe(pipe_class_default);
7832 %}
7833
7834 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7835 match(Set dst (PopCountI (LoadI mem)));
7836 effect(TEMP tmp);
7837 ins_cost(INSN_COST * 13);
7838
7839 format %{ "ldrs $tmp, $mem\n\t"
7840 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7841 "addv $tmp, $tmp\t# vector (8B)\n\t"
7842 "mov $dst, $tmp\t# vector (1D)" %}
7843 ins_encode %{
7844 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7845 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7846 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7847 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7848 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7849 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7850 %}
7851
7852 ins_pipe(pipe_class_default);
7853 %}
7854
7855 // Note: Long.bitCount(long) returns an int.
7856 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7857 match(Set dst (PopCountL src));
7858 effect(TEMP tmp);
7859 ins_cost(INSN_COST * 13);
7860
7861 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7862 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7863 "addv $tmp, $tmp\t# vector (8B)\n\t"
7864 "mov $dst, $tmp\t# vector (1D)" %}
7865 ins_encode %{
7866 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7867 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7868 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7869 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7870 %}
7871
7872 ins_pipe(pipe_class_default);
7873 %}
7874
7875 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7876 match(Set dst (PopCountL (LoadL mem)));
7877 effect(TEMP tmp);
7878 ins_cost(INSN_COST * 13);
7879
7880 format %{ "ldrd $tmp, $mem\n\t"
7881 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7882 "addv $tmp, $tmp\t# vector (8B)\n\t"
7883 "mov $dst, $tmp\t# vector (1D)" %}
7884 ins_encode %{
7885 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7886 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7888 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7889 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7890 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7891 %}
7892
7893 ins_pipe(pipe_class_default);
7894 %}
7895
7896 // ============================================================================
7897 // VerifyVectorAlignment Instruction
7898
7899 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7900 match(Set addr (VerifyVectorAlignment addr mask));
7901 effect(KILL cr);
7902 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7903 ins_encode %{
7904 Label Lskip;
7905 // check if masked bits of addr are zero
7906 __ tst($addr$$Register, $mask$$constant);
7907 __ br(Assembler::EQ, Lskip);
7908 __ stop("verify_vector_alignment found a misaligned vector memory access");
7909 __ bind(Lskip);
7910 %}
7911 ins_pipe(pipe_slow);
7912 %}
7913
7914 // ============================================================================
7915 // MemBar Instruction
7916
7917 instruct load_fence() %{
7918 match(LoadFence);
7919 ins_cost(VOLATILE_REF_COST);
7920
7921 format %{ "load_fence" %}
7922
7923 ins_encode %{
7924 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7925 %}
7926 ins_pipe(pipe_serial);
7927 %}
7928
7929 instruct unnecessary_membar_acquire() %{
7930 predicate(unnecessary_acquire(n));
7931 match(MemBarAcquire);
7932 ins_cost(0);
7933
7934 format %{ "membar_acquire (elided)" %}
7935
7936 ins_encode %{
7937 __ block_comment("membar_acquire (elided)");
7938 %}
7939
7940 ins_pipe(pipe_class_empty);
7941 %}
7942
7943 instruct membar_acquire() %{
7944 match(MemBarAcquire);
7945 ins_cost(VOLATILE_REF_COST);
7946
7947 format %{ "membar_acquire\n\t"
7948 "dmb ishld" %}
7949
7950 ins_encode %{
7951 __ block_comment("membar_acquire");
7952 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7953 %}
7954
7955 ins_pipe(pipe_serial);
7956 %}
7957
7958
7959 instruct membar_acquire_lock() %{
7960 match(MemBarAcquireLock);
7961 ins_cost(VOLATILE_REF_COST);
7962
7963 format %{ "membar_acquire_lock (elided)" %}
7964
7965 ins_encode %{
7966 __ block_comment("membar_acquire_lock (elided)");
7967 %}
7968
7969 ins_pipe(pipe_serial);
7970 %}
7971
7972 instruct store_fence() %{
7973 match(StoreFence);
7974 ins_cost(VOLATILE_REF_COST);
7975
7976 format %{ "store_fence" %}
7977
7978 ins_encode %{
7979 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7980 %}
7981 ins_pipe(pipe_serial);
7982 %}
7983
7984 instruct unnecessary_membar_release() %{
7985 predicate(unnecessary_release(n));
7986 match(MemBarRelease);
7987 ins_cost(0);
7988
7989 format %{ "membar_release (elided)" %}
7990
7991 ins_encode %{
7992 __ block_comment("membar_release (elided)");
7993 %}
7994 ins_pipe(pipe_serial);
7995 %}
7996
7997 instruct membar_release() %{
7998 match(MemBarRelease);
7999 ins_cost(VOLATILE_REF_COST);
8000
8001 format %{ "membar_release\n\t"
8002 "dmb ishst\n\tdmb ishld" %}
8003
8004 ins_encode %{
8005 __ block_comment("membar_release");
8006 // These will be merged if AlwaysMergeDMB is enabled.
8007 __ membar(Assembler::StoreStore);
8008 __ membar(Assembler::LoadStore);
8009 %}
8010 ins_pipe(pipe_serial);
8011 %}
8012
8013 instruct membar_storestore() %{
8014 match(MemBarStoreStore);
8015 match(StoreStoreFence);
8016 ins_cost(VOLATILE_REF_COST);
8017
8018 format %{ "MEMBAR-store-store" %}
8019
8020 ins_encode %{
8021 __ membar(Assembler::StoreStore);
8022 %}
8023 ins_pipe(pipe_serial);
8024 %}
8025
8026 instruct membar_release_lock() %{
8027 match(MemBarReleaseLock);
8028 ins_cost(VOLATILE_REF_COST);
8029
8030 format %{ "membar_release_lock (elided)" %}
8031
8032 ins_encode %{
8033 __ block_comment("membar_release_lock (elided)");
8034 %}
8035
8036 ins_pipe(pipe_serial);
8037 %}
8038
8039 instruct unnecessary_membar_volatile() %{
8040 predicate(unnecessary_volatile(n));
8041 match(MemBarVolatile);
8042 ins_cost(0);
8043
8044 format %{ "membar_volatile (elided)" %}
8045
8046 ins_encode %{
8047 __ block_comment("membar_volatile (elided)");
8048 %}
8049
8050 ins_pipe(pipe_serial);
8051 %}
8052
8053 instruct membar_volatile() %{
8054 match(MemBarVolatile);
8055 ins_cost(VOLATILE_REF_COST*100);
8056
8057 format %{ "membar_volatile\n\t"
8058 "dmb ish"%}
8059
8060 ins_encode %{
8061 __ block_comment("membar_volatile");
8062 __ membar(Assembler::StoreLoad);
8063 %}
8064
8065 ins_pipe(pipe_serial);
8066 %}
8067
8068 // ============================================================================
8069 // Cast/Convert Instructions
8070
8071 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8072 match(Set dst (CastX2P src));
8073
8074 ins_cost(INSN_COST);
8075 format %{ "mov $dst, $src\t# long -> ptr" %}
8076
8077 ins_encode %{
8078 if ($dst$$reg != $src$$reg) {
8079 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8080 }
8081 %}
8082
8083 ins_pipe(ialu_reg);
8084 %}
8085
8086 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8087 match(Set dst (CastP2X src));
8088
8089 ins_cost(INSN_COST);
8090 format %{ "mov $dst, $src\t# ptr -> long" %}
8091
8092 ins_encode %{
8093 if ($dst$$reg != $src$$reg) {
8094 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8095 }
8096 %}
8097
8098 ins_pipe(ialu_reg);
8099 %}
8100
8101 // Convert oop into int for vectors alignment masking
8102 instruct convP2I(iRegINoSp dst, iRegP src) %{
8103 match(Set dst (ConvL2I (CastP2X src)));
8104
8105 ins_cost(INSN_COST);
8106 format %{ "movw $dst, $src\t# ptr -> int" %}
8107 ins_encode %{
8108 __ movw($dst$$Register, $src$$Register);
8109 %}
8110
8111 ins_pipe(ialu_reg);
8112 %}
8113
8114 // Convert compressed oop into int for vectors alignment masking
8115 // in case of 32bit oops (heap < 4Gb).
8116 instruct convN2I(iRegINoSp dst, iRegN src)
8117 %{
8118 predicate(CompressedOops::shift() == 0);
8119 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8120
8121 ins_cost(INSN_COST);
8122 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8123 ins_encode %{
8124 __ movw($dst$$Register, $src$$Register);
8125 %}
8126
8127 ins_pipe(ialu_reg);
8128 %}
8129
8130
8131 // Convert oop pointer into compressed form
8132 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8133 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8134 match(Set dst (EncodeP src));
8135 effect(KILL cr);
8136 ins_cost(INSN_COST * 3);
8137 format %{ "encode_heap_oop $dst, $src" %}
8138 ins_encode %{
8139 Register s = $src$$Register;
8140 Register d = $dst$$Register;
8141 __ encode_heap_oop(d, s);
8142 %}
8143 ins_pipe(ialu_reg);
8144 %}
8145
8146 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8147 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8148 match(Set dst (EncodeP src));
8149 ins_cost(INSN_COST * 3);
8150 format %{ "encode_heap_oop_not_null $dst, $src" %}
8151 ins_encode %{
8152 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8153 %}
8154 ins_pipe(ialu_reg);
8155 %}
8156
8157 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8158 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8159 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8160 match(Set dst (DecodeN src));
8161 ins_cost(INSN_COST * 3);
8162 format %{ "decode_heap_oop $dst, $src" %}
8163 ins_encode %{
8164 Register s = $src$$Register;
8165 Register d = $dst$$Register;
8166 __ decode_heap_oop(d, s);
8167 %}
8168 ins_pipe(ialu_reg);
8169 %}
8170
8171 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8172 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8173 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8174 match(Set dst (DecodeN src));
8175 ins_cost(INSN_COST * 3);
8176 format %{ "decode_heap_oop_not_null $dst, $src" %}
8177 ins_encode %{
8178 Register s = $src$$Register;
8179 Register d = $dst$$Register;
8180 __ decode_heap_oop_not_null(d, s);
8181 %}
8182 ins_pipe(ialu_reg);
8183 %}
8184
8185 // n.b. AArch64 implementations of encode_klass_not_null and
8186 // decode_klass_not_null do not modify the flags register so, unlike
8187 // Intel, we don't kill CR as a side effect here
8188
8189 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8190 match(Set dst (EncodePKlass src));
8191
8192 ins_cost(INSN_COST * 3);
8193 format %{ "encode_klass_not_null $dst,$src" %}
8194
8195 ins_encode %{
8196 Register src_reg = as_Register($src$$reg);
8197 Register dst_reg = as_Register($dst$$reg);
8198 __ encode_klass_not_null(dst_reg, src_reg);
8199 %}
8200
8201 ins_pipe(ialu_reg);
8202 %}
8203
8204 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8205 match(Set dst (DecodeNKlass src));
8206
8207 ins_cost(INSN_COST * 3);
8208 format %{ "decode_klass_not_null $dst,$src" %}
8209
8210 ins_encode %{
8211 Register src_reg = as_Register($src$$reg);
8212 Register dst_reg = as_Register($dst$$reg);
8213 if (dst_reg != src_reg) {
8214 __ decode_klass_not_null(dst_reg, src_reg);
8215 } else {
8216 __ decode_klass_not_null(dst_reg);
8217 }
8218 %}
8219
8220 ins_pipe(ialu_reg);
8221 %}
8222
8223 instruct checkCastPP(iRegPNoSp dst)
8224 %{
8225 match(Set dst (CheckCastPP dst));
8226
8227 size(0);
8228 format %{ "# checkcastPP of $dst" %}
8229 ins_encode(/* empty encoding */);
8230 ins_pipe(pipe_class_empty);
8231 %}
8232
8233 instruct castPP(iRegPNoSp dst)
8234 %{
8235 match(Set dst (CastPP dst));
8236
8237 size(0);
8238 format %{ "# castPP of $dst" %}
8239 ins_encode(/* empty encoding */);
8240 ins_pipe(pipe_class_empty);
8241 %}
8242
8243 instruct castII(iRegI dst)
8244 %{
8245 predicate(VerifyConstraintCasts == 0);
8246 match(Set dst (CastII dst));
8247
8248 size(0);
8249 format %{ "# castII of $dst" %}
8250 ins_encode(/* empty encoding */);
8251 ins_cost(0);
8252 ins_pipe(pipe_class_empty);
8253 %}
8254
8255 instruct castII_checked(iRegI dst, rFlagsReg cr)
8256 %{
8257 predicate(VerifyConstraintCasts > 0);
8258 match(Set dst (CastII dst));
8259 effect(KILL cr);
8260
8261 format %{ "# castII_checked of $dst" %}
8262 ins_encode %{
8263 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8264 %}
8265 ins_pipe(pipe_slow);
8266 %}
8267
8268 instruct castLL(iRegL dst)
8269 %{
8270 predicate(VerifyConstraintCasts == 0);
8271 match(Set dst (CastLL dst));
8272
8273 size(0);
8274 format %{ "# castLL of $dst" %}
8275 ins_encode(/* empty encoding */);
8276 ins_cost(0);
8277 ins_pipe(pipe_class_empty);
8278 %}
8279
8280 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8281 %{
8282 predicate(VerifyConstraintCasts > 0);
8283 match(Set dst (CastLL dst));
8284 effect(KILL cr);
8285
8286 format %{ "# castLL_checked of $dst" %}
8287 ins_encode %{
8288 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8289 %}
8290 ins_pipe(pipe_slow);
8291 %}
8292
8293 instruct castHH(vRegF dst)
8294 %{
8295 match(Set dst (CastHH dst));
8296 size(0);
8297 format %{ "# castHH of $dst" %}
8298 ins_encode(/* empty encoding */);
8299 ins_cost(0);
8300 ins_pipe(pipe_class_empty);
8301 %}
8302
8303 instruct castFF(vRegF dst)
8304 %{
8305 match(Set dst (CastFF dst));
8306
8307 size(0);
8308 format %{ "# castFF of $dst" %}
8309 ins_encode(/* empty encoding */);
8310 ins_cost(0);
8311 ins_pipe(pipe_class_empty);
8312 %}
8313
8314 instruct castDD(vRegD dst)
8315 %{
8316 match(Set dst (CastDD dst));
8317
8318 size(0);
8319 format %{ "# castDD of $dst" %}
8320 ins_encode(/* empty encoding */);
8321 ins_cost(0);
8322 ins_pipe(pipe_class_empty);
8323 %}
8324
8325 instruct castVV(vReg dst)
8326 %{
8327 match(Set dst (CastVV dst));
8328
8329 size(0);
8330 format %{ "# castVV of $dst" %}
8331 ins_encode(/* empty encoding */);
8332 ins_cost(0);
8333 ins_pipe(pipe_class_empty);
8334 %}
8335
8336 instruct castVVMask(pRegGov dst)
8337 %{
8338 match(Set dst (CastVV dst));
8339
8340 size(0);
8341 format %{ "# castVV of $dst" %}
8342 ins_encode(/* empty encoding */);
8343 ins_cost(0);
8344 ins_pipe(pipe_class_empty);
8345 %}
8346
8347 // Manifest a CmpU result in an integer register.
8348 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8349 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8350 %{
8351 match(Set dst (CmpU3 src1 src2));
8352 effect(KILL flags);
8353
8354 ins_cost(INSN_COST * 3);
8355 format %{
8356 "cmpw $src1, $src2\n\t"
8357 "csetw $dst, ne\n\t"
8358 "cnegw $dst, lo\t# CmpU3(reg)"
8359 %}
8360 ins_encode %{
8361 __ cmpw($src1$$Register, $src2$$Register);
8362 __ csetw($dst$$Register, Assembler::NE);
8363 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8364 %}
8365
8366 ins_pipe(pipe_class_default);
8367 %}
8368
8369 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8370 %{
8371 match(Set dst (CmpU3 src1 src2));
8372 effect(KILL flags);
8373
8374 ins_cost(INSN_COST * 3);
8375 format %{
8376 "subsw zr, $src1, $src2\n\t"
8377 "csetw $dst, ne\n\t"
8378 "cnegw $dst, lo\t# CmpU3(imm)"
8379 %}
8380 ins_encode %{
8381 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8382 __ csetw($dst$$Register, Assembler::NE);
8383 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8384 %}
8385
8386 ins_pipe(pipe_class_default);
8387 %}
8388
8389 // Manifest a CmpUL result in an integer register.
8390 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8391 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8392 %{
8393 match(Set dst (CmpUL3 src1 src2));
8394 effect(KILL flags);
8395
8396 ins_cost(INSN_COST * 3);
8397 format %{
8398 "cmp $src1, $src2\n\t"
8399 "csetw $dst, ne\n\t"
8400 "cnegw $dst, lo\t# CmpUL3(reg)"
8401 %}
8402 ins_encode %{
8403 __ cmp($src1$$Register, $src2$$Register);
8404 __ csetw($dst$$Register, Assembler::NE);
8405 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8406 %}
8407
8408 ins_pipe(pipe_class_default);
8409 %}
8410
8411 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8412 %{
8413 match(Set dst (CmpUL3 src1 src2));
8414 effect(KILL flags);
8415
8416 ins_cost(INSN_COST * 3);
8417 format %{
8418 "subs zr, $src1, $src2\n\t"
8419 "csetw $dst, ne\n\t"
8420 "cnegw $dst, lo\t# CmpUL3(imm)"
8421 %}
8422 ins_encode %{
8423 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8424 __ csetw($dst$$Register, Assembler::NE);
8425 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8426 %}
8427
8428 ins_pipe(pipe_class_default);
8429 %}
8430
8431 // Manifest a CmpL result in an integer register.
8432 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8433 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8434 %{
8435 match(Set dst (CmpL3 src1 src2));
8436 effect(KILL flags);
8437
8438 ins_cost(INSN_COST * 3);
8439 format %{
8440 "cmp $src1, $src2\n\t"
8441 "csetw $dst, ne\n\t"
8442 "cnegw $dst, lt\t# CmpL3(reg)"
8443 %}
8444 ins_encode %{
8445 __ cmp($src1$$Register, $src2$$Register);
8446 __ csetw($dst$$Register, Assembler::NE);
8447 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8448 %}
8449
8450 ins_pipe(pipe_class_default);
8451 %}
8452
8453 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8454 %{
8455 match(Set dst (CmpL3 src1 src2));
8456 effect(KILL flags);
8457
8458 ins_cost(INSN_COST * 3);
8459 format %{
8460 "subs zr, $src1, $src2\n\t"
8461 "csetw $dst, ne\n\t"
8462 "cnegw $dst, lt\t# CmpL3(imm)"
8463 %}
8464 ins_encode %{
8465 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8466 __ csetw($dst$$Register, Assembler::NE);
8467 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8468 %}
8469
8470 ins_pipe(pipe_class_default);
8471 %}
8472
8473 // ============================================================================
8474 // Conditional Move Instructions
8475
8476 // n.b. we have identical rules for both a signed compare op (cmpOp)
8477 // and an unsigned compare op (cmpOpU). it would be nice if we could
8478 // define an op class which merged both inputs and use it to type the
8479 // argument to a single rule. unfortunatelyt his fails because the
8480 // opclass does not live up to the COND_INTER interface of its
8481 // component operands. When the generic code tries to negate the
8482 // operand it ends up running the generci Machoper::negate method
8483 // which throws a ShouldNotHappen. So, we have to provide two flavours
8484 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8485
8486 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8487 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8488
8489 ins_cost(INSN_COST * 2);
8490 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8491
8492 ins_encode %{
8493 __ cselw(as_Register($dst$$reg),
8494 as_Register($src2$$reg),
8495 as_Register($src1$$reg),
8496 (Assembler::Condition)$cmp$$cmpcode);
8497 %}
8498
8499 ins_pipe(icond_reg_reg);
8500 %}
8501
8502 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8503 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8504
8505 ins_cost(INSN_COST * 2);
8506 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8507
8508 ins_encode %{
8509 __ cselw(as_Register($dst$$reg),
8510 as_Register($src2$$reg),
8511 as_Register($src1$$reg),
8512 (Assembler::Condition)$cmp$$cmpcode);
8513 %}
8514
8515 ins_pipe(icond_reg_reg);
8516 %}
8517
8518 // special cases where one arg is zero
8519
8520 // n.b. this is selected in preference to the rule above because it
8521 // avoids loading constant 0 into a source register
8522
8523 // TODO
8524 // we ought only to be able to cull one of these variants as the ideal
8525 // transforms ought always to order the zero consistently (to left/right?)
8526
8527 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8528 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8529
8530 ins_cost(INSN_COST * 2);
8531 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8532
8533 ins_encode %{
8534 __ cselw(as_Register($dst$$reg),
8535 as_Register($src$$reg),
8536 zr,
8537 (Assembler::Condition)$cmp$$cmpcode);
8538 %}
8539
8540 ins_pipe(icond_reg);
8541 %}
8542
8543 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8544 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8545
8546 ins_cost(INSN_COST * 2);
8547 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8548
8549 ins_encode %{
8550 __ cselw(as_Register($dst$$reg),
8551 as_Register($src$$reg),
8552 zr,
8553 (Assembler::Condition)$cmp$$cmpcode);
8554 %}
8555
8556 ins_pipe(icond_reg);
8557 %}
8558
8559 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8560 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8561
8562 ins_cost(INSN_COST * 2);
8563 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8564
8565 ins_encode %{
8566 __ cselw(as_Register($dst$$reg),
8567 zr,
8568 as_Register($src$$reg),
8569 (Assembler::Condition)$cmp$$cmpcode);
8570 %}
8571
8572 ins_pipe(icond_reg);
8573 %}
8574
8575 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8576 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8577
8578 ins_cost(INSN_COST * 2);
8579 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8580
8581 ins_encode %{
8582 __ cselw(as_Register($dst$$reg),
8583 zr,
8584 as_Register($src$$reg),
8585 (Assembler::Condition)$cmp$$cmpcode);
8586 %}
8587
8588 ins_pipe(icond_reg);
8589 %}
8590
8591 // special case for creating a boolean 0 or 1
8592
8593 // n.b. this is selected in preference to the rule above because it
8594 // avoids loading constants 0 and 1 into a source register
8595
8596 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8597 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8598
8599 ins_cost(INSN_COST * 2);
8600 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8601
8602 ins_encode %{
8603 // equivalently
8604 // cset(as_Register($dst$$reg),
8605 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8606 __ csincw(as_Register($dst$$reg),
8607 zr,
8608 zr,
8609 (Assembler::Condition)$cmp$$cmpcode);
8610 %}
8611
8612 ins_pipe(icond_none);
8613 %}
8614
8615 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8616 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8617
8618 ins_cost(INSN_COST * 2);
8619 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8620
8621 ins_encode %{
8622 // equivalently
8623 // cset(as_Register($dst$$reg),
8624 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8625 __ csincw(as_Register($dst$$reg),
8626 zr,
8627 zr,
8628 (Assembler::Condition)$cmp$$cmpcode);
8629 %}
8630
8631 ins_pipe(icond_none);
8632 %}
8633
8634 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8635 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8636
8637 ins_cost(INSN_COST * 2);
8638 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8639
8640 ins_encode %{
8641 __ csel(as_Register($dst$$reg),
8642 as_Register($src2$$reg),
8643 as_Register($src1$$reg),
8644 (Assembler::Condition)$cmp$$cmpcode);
8645 %}
8646
8647 ins_pipe(icond_reg_reg);
8648 %}
8649
8650 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8651 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8652
8653 ins_cost(INSN_COST * 2);
8654 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8655
8656 ins_encode %{
8657 __ csel(as_Register($dst$$reg),
8658 as_Register($src2$$reg),
8659 as_Register($src1$$reg),
8660 (Assembler::Condition)$cmp$$cmpcode);
8661 %}
8662
8663 ins_pipe(icond_reg_reg);
8664 %}
8665
8666 // special cases where one arg is zero
8667
8668 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8669 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8670
8671 ins_cost(INSN_COST * 2);
8672 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8673
8674 ins_encode %{
8675 __ csel(as_Register($dst$$reg),
8676 zr,
8677 as_Register($src$$reg),
8678 (Assembler::Condition)$cmp$$cmpcode);
8679 %}
8680
8681 ins_pipe(icond_reg);
8682 %}
8683
8684 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8685 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8686
8687 ins_cost(INSN_COST * 2);
8688 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8689
8690 ins_encode %{
8691 __ csel(as_Register($dst$$reg),
8692 zr,
8693 as_Register($src$$reg),
8694 (Assembler::Condition)$cmp$$cmpcode);
8695 %}
8696
8697 ins_pipe(icond_reg);
8698 %}
8699
8700 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8701 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8702
8703 ins_cost(INSN_COST * 2);
8704 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8705
8706 ins_encode %{
8707 __ csel(as_Register($dst$$reg),
8708 as_Register($src$$reg),
8709 zr,
8710 (Assembler::Condition)$cmp$$cmpcode);
8711 %}
8712
8713 ins_pipe(icond_reg);
8714 %}
8715
8716 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8717 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8718
8719 ins_cost(INSN_COST * 2);
8720 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8721
8722 ins_encode %{
8723 __ csel(as_Register($dst$$reg),
8724 as_Register($src$$reg),
8725 zr,
8726 (Assembler::Condition)$cmp$$cmpcode);
8727 %}
8728
8729 ins_pipe(icond_reg);
8730 %}
8731
8732 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8733 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8734
8735 ins_cost(INSN_COST * 2);
8736 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8737
8738 ins_encode %{
8739 __ csel(as_Register($dst$$reg),
8740 as_Register($src2$$reg),
8741 as_Register($src1$$reg),
8742 (Assembler::Condition)$cmp$$cmpcode);
8743 %}
8744
8745 ins_pipe(icond_reg_reg);
8746 %}
8747
8748 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8749 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8750
8751 ins_cost(INSN_COST * 2);
8752 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8753
8754 ins_encode %{
8755 __ csel(as_Register($dst$$reg),
8756 as_Register($src2$$reg),
8757 as_Register($src1$$reg),
8758 (Assembler::Condition)$cmp$$cmpcode);
8759 %}
8760
8761 ins_pipe(icond_reg_reg);
8762 %}
8763
8764 // special cases where one arg is zero
8765
8766 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8767 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8768
8769 ins_cost(INSN_COST * 2);
8770 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8771
8772 ins_encode %{
8773 __ csel(as_Register($dst$$reg),
8774 zr,
8775 as_Register($src$$reg),
8776 (Assembler::Condition)$cmp$$cmpcode);
8777 %}
8778
8779 ins_pipe(icond_reg);
8780 %}
8781
8782 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8783 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8784
8785 ins_cost(INSN_COST * 2);
8786 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8787
8788 ins_encode %{
8789 __ csel(as_Register($dst$$reg),
8790 zr,
8791 as_Register($src$$reg),
8792 (Assembler::Condition)$cmp$$cmpcode);
8793 %}
8794
8795 ins_pipe(icond_reg);
8796 %}
8797
8798 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8799 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8800
8801 ins_cost(INSN_COST * 2);
8802 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8803
8804 ins_encode %{
8805 __ csel(as_Register($dst$$reg),
8806 as_Register($src$$reg),
8807 zr,
8808 (Assembler::Condition)$cmp$$cmpcode);
8809 %}
8810
8811 ins_pipe(icond_reg);
8812 %}
8813
8814 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8815 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8816
8817 ins_cost(INSN_COST * 2);
8818 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8819
8820 ins_encode %{
8821 __ csel(as_Register($dst$$reg),
8822 as_Register($src$$reg),
8823 zr,
8824 (Assembler::Condition)$cmp$$cmpcode);
8825 %}
8826
8827 ins_pipe(icond_reg);
8828 %}
8829
8830 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8831 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8832
8833 ins_cost(INSN_COST * 2);
8834 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8835
8836 ins_encode %{
8837 __ cselw(as_Register($dst$$reg),
8838 as_Register($src2$$reg),
8839 as_Register($src1$$reg),
8840 (Assembler::Condition)$cmp$$cmpcode);
8841 %}
8842
8843 ins_pipe(icond_reg_reg);
8844 %}
8845
8846 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8847 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8848
8849 ins_cost(INSN_COST * 2);
8850 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8851
8852 ins_encode %{
8853 __ cselw(as_Register($dst$$reg),
8854 as_Register($src2$$reg),
8855 as_Register($src1$$reg),
8856 (Assembler::Condition)$cmp$$cmpcode);
8857 %}
8858
8859 ins_pipe(icond_reg_reg);
8860 %}
8861
8862 // special cases where one arg is zero
8863
8864 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8865 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8866
8867 ins_cost(INSN_COST * 2);
8868 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8869
8870 ins_encode %{
8871 __ cselw(as_Register($dst$$reg),
8872 zr,
8873 as_Register($src$$reg),
8874 (Assembler::Condition)$cmp$$cmpcode);
8875 %}
8876
8877 ins_pipe(icond_reg);
8878 %}
8879
8880 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8881 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8882
8883 ins_cost(INSN_COST * 2);
8884 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8885
8886 ins_encode %{
8887 __ cselw(as_Register($dst$$reg),
8888 zr,
8889 as_Register($src$$reg),
8890 (Assembler::Condition)$cmp$$cmpcode);
8891 %}
8892
8893 ins_pipe(icond_reg);
8894 %}
8895
8896 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8897 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8898
8899 ins_cost(INSN_COST * 2);
8900 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8901
8902 ins_encode %{
8903 __ cselw(as_Register($dst$$reg),
8904 as_Register($src$$reg),
8905 zr,
8906 (Assembler::Condition)$cmp$$cmpcode);
8907 %}
8908
8909 ins_pipe(icond_reg);
8910 %}
8911
8912 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8913 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8914
8915 ins_cost(INSN_COST * 2);
8916 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8917
8918 ins_encode %{
8919 __ cselw(as_Register($dst$$reg),
8920 as_Register($src$$reg),
8921 zr,
8922 (Assembler::Condition)$cmp$$cmpcode);
8923 %}
8924
8925 ins_pipe(icond_reg);
8926 %}
8927
8928 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8929 %{
8930 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8931
8932 ins_cost(INSN_COST * 3);
8933
8934 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8935 ins_encode %{
8936 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8937 __ fcsels(as_FloatRegister($dst$$reg),
8938 as_FloatRegister($src2$$reg),
8939 as_FloatRegister($src1$$reg),
8940 cond);
8941 %}
8942
8943 ins_pipe(fp_cond_reg_reg_s);
8944 %}
8945
8946 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8947 %{
8948 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8949
8950 ins_cost(INSN_COST * 3);
8951
8952 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8953 ins_encode %{
8954 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8955 __ fcsels(as_FloatRegister($dst$$reg),
8956 as_FloatRegister($src2$$reg),
8957 as_FloatRegister($src1$$reg),
8958 cond);
8959 %}
8960
8961 ins_pipe(fp_cond_reg_reg_s);
8962 %}
8963
8964 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8965 %{
8966 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8967
8968 ins_cost(INSN_COST * 3);
8969
8970 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8971 ins_encode %{
8972 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8973 __ fcseld(as_FloatRegister($dst$$reg),
8974 as_FloatRegister($src2$$reg),
8975 as_FloatRegister($src1$$reg),
8976 cond);
8977 %}
8978
8979 ins_pipe(fp_cond_reg_reg_d);
8980 %}
8981
8982 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8983 %{
8984 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8985
8986 ins_cost(INSN_COST * 3);
8987
8988 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8989 ins_encode %{
8990 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8991 __ fcseld(as_FloatRegister($dst$$reg),
8992 as_FloatRegister($src2$$reg),
8993 as_FloatRegister($src1$$reg),
8994 cond);
8995 %}
8996
8997 ins_pipe(fp_cond_reg_reg_d);
8998 %}
8999
9000 // ============================================================================
9001 // Arithmetic Instructions
9002 //
9003
9004 // Integer Addition
9005
9006 // TODO
9007 // these currently employ operations which do not set CR and hence are
9008 // not flagged as killing CR but we would like to isolate the cases
9009 // where we want to set flags from those where we don't. need to work
9010 // out how to do that.
9011
9012 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9013 match(Set dst (AddI src1 src2));
9014
9015 ins_cost(INSN_COST);
9016 format %{ "addw $dst, $src1, $src2" %}
9017
9018 ins_encode %{
9019 __ addw(as_Register($dst$$reg),
9020 as_Register($src1$$reg),
9021 as_Register($src2$$reg));
9022 %}
9023
9024 ins_pipe(ialu_reg_reg);
9025 %}
9026
9027 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9028 match(Set dst (AddI src1 src2));
9029
9030 ins_cost(INSN_COST);
9031 format %{ "addw $dst, $src1, $src2" %}
9032
9033 // use opcode to indicate that this is an add not a sub
9034 opcode(0x0);
9035
9036 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9037
9038 ins_pipe(ialu_reg_imm);
9039 %}
9040
9041 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9042 match(Set dst (AddI (ConvL2I src1) src2));
9043
9044 ins_cost(INSN_COST);
9045 format %{ "addw $dst, $src1, $src2" %}
9046
9047 // use opcode to indicate that this is an add not a sub
9048 opcode(0x0);
9049
9050 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9051
9052 ins_pipe(ialu_reg_imm);
9053 %}
9054
9055 // Pointer Addition
9056 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9057 match(Set dst (AddP src1 src2));
9058
9059 ins_cost(INSN_COST);
9060 format %{ "add $dst, $src1, $src2\t# ptr" %}
9061
9062 ins_encode %{
9063 __ add(as_Register($dst$$reg),
9064 as_Register($src1$$reg),
9065 as_Register($src2$$reg));
9066 %}
9067
9068 ins_pipe(ialu_reg_reg);
9069 %}
9070
9071 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9072 match(Set dst (AddP src1 (ConvI2L src2)));
9073
9074 ins_cost(1.9 * INSN_COST);
9075 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9076
9077 ins_encode %{
9078 __ add(as_Register($dst$$reg),
9079 as_Register($src1$$reg),
9080 as_Register($src2$$reg), ext::sxtw);
9081 %}
9082
9083 ins_pipe(ialu_reg_reg);
9084 %}
9085
9086 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9087 match(Set dst (AddP src1 (LShiftL src2 scale)));
9088
9089 ins_cost(1.9 * INSN_COST);
9090 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9091
9092 ins_encode %{
9093 __ lea(as_Register($dst$$reg),
9094 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9095 Address::lsl($scale$$constant)));
9096 %}
9097
9098 ins_pipe(ialu_reg_reg_shift);
9099 %}
9100
9101 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9102 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9103
9104 ins_cost(1.9 * INSN_COST);
9105 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9106
9107 ins_encode %{
9108 __ lea(as_Register($dst$$reg),
9109 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9110 Address::sxtw($scale$$constant)));
9111 %}
9112
9113 ins_pipe(ialu_reg_reg_shift);
9114 %}
9115
9116 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9117 match(Set dst (LShiftL (ConvI2L src) scale));
9118
9119 ins_cost(INSN_COST);
9120 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9121
9122 ins_encode %{
9123 __ sbfiz(as_Register($dst$$reg),
9124 as_Register($src$$reg),
9125 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9126 %}
9127
9128 ins_pipe(ialu_reg_shift);
9129 %}
9130
9131 // Pointer Immediate Addition
9132 // n.b. this needs to be more expensive than using an indirect memory
9133 // operand
9134 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9135 match(Set dst (AddP src1 src2));
9136
9137 ins_cost(INSN_COST);
9138 format %{ "add $dst, $src1, $src2\t# ptr" %}
9139
9140 // use opcode to indicate that this is an add not a sub
9141 opcode(0x0);
9142
9143 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9144
9145 ins_pipe(ialu_reg_imm);
9146 %}
9147
9148 // Long Addition
9149 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9150
9151 match(Set dst (AddL src1 src2));
9152
9153 ins_cost(INSN_COST);
9154 format %{ "add $dst, $src1, $src2" %}
9155
9156 ins_encode %{
9157 __ add(as_Register($dst$$reg),
9158 as_Register($src1$$reg),
9159 as_Register($src2$$reg));
9160 %}
9161
9162 ins_pipe(ialu_reg_reg);
9163 %}
9164
9165 // No constant pool entries requiredLong Immediate Addition.
9166 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9167 match(Set dst (AddL src1 src2));
9168
9169 ins_cost(INSN_COST);
9170 format %{ "add $dst, $src1, $src2" %}
9171
9172 // use opcode to indicate that this is an add not a sub
9173 opcode(0x0);
9174
9175 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9176
9177 ins_pipe(ialu_reg_imm);
9178 %}
9179
9180 // Integer Subtraction
9181 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9182 match(Set dst (SubI src1 src2));
9183
9184 ins_cost(INSN_COST);
9185 format %{ "subw $dst, $src1, $src2" %}
9186
9187 ins_encode %{
9188 __ subw(as_Register($dst$$reg),
9189 as_Register($src1$$reg),
9190 as_Register($src2$$reg));
9191 %}
9192
9193 ins_pipe(ialu_reg_reg);
9194 %}
9195
9196 // Immediate Subtraction
9197 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9198 match(Set dst (SubI src1 src2));
9199
9200 ins_cost(INSN_COST);
9201 format %{ "subw $dst, $src1, $src2" %}
9202
9203 // use opcode to indicate that this is a sub not an add
9204 opcode(0x1);
9205
9206 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9207
9208 ins_pipe(ialu_reg_imm);
9209 %}
9210
9211 // Long Subtraction
9212 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9213
9214 match(Set dst (SubL src1 src2));
9215
9216 ins_cost(INSN_COST);
9217 format %{ "sub $dst, $src1, $src2" %}
9218
9219 ins_encode %{
9220 __ sub(as_Register($dst$$reg),
9221 as_Register($src1$$reg),
9222 as_Register($src2$$reg));
9223 %}
9224
9225 ins_pipe(ialu_reg_reg);
9226 %}
9227
9228 // No constant pool entries requiredLong Immediate Subtraction.
9229 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9230 match(Set dst (SubL src1 src2));
9231
9232 ins_cost(INSN_COST);
9233 format %{ "sub$dst, $src1, $src2" %}
9234
9235 // use opcode to indicate that this is a sub not an add
9236 opcode(0x1);
9237
9238 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9239
9240 ins_pipe(ialu_reg_imm);
9241 %}
9242
9243 // Integer Negation (special case for sub)
9244
9245 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9246 match(Set dst (SubI zero src));
9247
9248 ins_cost(INSN_COST);
9249 format %{ "negw $dst, $src\t# int" %}
9250
9251 ins_encode %{
9252 __ negw(as_Register($dst$$reg),
9253 as_Register($src$$reg));
9254 %}
9255
9256 ins_pipe(ialu_reg);
9257 %}
9258
9259 // Long Negation
9260
9261 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9262 match(Set dst (SubL zero src));
9263
9264 ins_cost(INSN_COST);
9265 format %{ "neg $dst, $src\t# long" %}
9266
9267 ins_encode %{
9268 __ neg(as_Register($dst$$reg),
9269 as_Register($src$$reg));
9270 %}
9271
9272 ins_pipe(ialu_reg);
9273 %}
9274
9275 // Integer Multiply
9276
9277 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9278 match(Set dst (MulI src1 src2));
9279
9280 ins_cost(INSN_COST * 3);
9281 format %{ "mulw $dst, $src1, $src2" %}
9282
9283 ins_encode %{
9284 __ mulw(as_Register($dst$$reg),
9285 as_Register($src1$$reg),
9286 as_Register($src2$$reg));
9287 %}
9288
9289 ins_pipe(imul_reg_reg);
9290 %}
9291
9292 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9293 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9294
9295 ins_cost(INSN_COST * 3);
9296 format %{ "smull $dst, $src1, $src2" %}
9297
9298 ins_encode %{
9299 __ smull(as_Register($dst$$reg),
9300 as_Register($src1$$reg),
9301 as_Register($src2$$reg));
9302 %}
9303
9304 ins_pipe(imul_reg_reg);
9305 %}
9306
9307 // Long Multiply
9308
9309 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9310 match(Set dst (MulL src1 src2));
9311
9312 ins_cost(INSN_COST * 5);
9313 format %{ "mul $dst, $src1, $src2" %}
9314
9315 ins_encode %{
9316 __ mul(as_Register($dst$$reg),
9317 as_Register($src1$$reg),
9318 as_Register($src2$$reg));
9319 %}
9320
9321 ins_pipe(lmul_reg_reg);
9322 %}
9323
9324 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9325 %{
9326 match(Set dst (MulHiL src1 src2));
9327
9328 ins_cost(INSN_COST * 7);
9329 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9330
9331 ins_encode %{
9332 __ smulh(as_Register($dst$$reg),
9333 as_Register($src1$$reg),
9334 as_Register($src2$$reg));
9335 %}
9336
9337 ins_pipe(lmul_reg_reg);
9338 %}
9339
9340 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9341 %{
9342 match(Set dst (UMulHiL src1 src2));
9343
9344 ins_cost(INSN_COST * 7);
9345 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9346
9347 ins_encode %{
9348 __ umulh(as_Register($dst$$reg),
9349 as_Register($src1$$reg),
9350 as_Register($src2$$reg));
9351 %}
9352
9353 ins_pipe(lmul_reg_reg);
9354 %}
9355
9356 // Combined Integer Multiply & Add/Sub
9357
9358 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9359 match(Set dst (AddI src3 (MulI src1 src2)));
9360
9361 ins_cost(INSN_COST * 3);
9362 format %{ "madd $dst, $src1, $src2, $src3" %}
9363
9364 ins_encode %{
9365 __ maddw(as_Register($dst$$reg),
9366 as_Register($src1$$reg),
9367 as_Register($src2$$reg),
9368 as_Register($src3$$reg));
9369 %}
9370
9371 ins_pipe(imac_reg_reg);
9372 %}
9373
9374 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9375 match(Set dst (SubI src3 (MulI src1 src2)));
9376
9377 ins_cost(INSN_COST * 3);
9378 format %{ "msub $dst, $src1, $src2, $src3" %}
9379
9380 ins_encode %{
9381 __ msubw(as_Register($dst$$reg),
9382 as_Register($src1$$reg),
9383 as_Register($src2$$reg),
9384 as_Register($src3$$reg));
9385 %}
9386
9387 ins_pipe(imac_reg_reg);
9388 %}
9389
9390 // Combined Integer Multiply & Neg
9391
9392 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9393 match(Set dst (MulI (SubI zero src1) src2));
9394
9395 ins_cost(INSN_COST * 3);
9396 format %{ "mneg $dst, $src1, $src2" %}
9397
9398 ins_encode %{
9399 __ mnegw(as_Register($dst$$reg),
9400 as_Register($src1$$reg),
9401 as_Register($src2$$reg));
9402 %}
9403
9404 ins_pipe(imac_reg_reg);
9405 %}
9406
9407 // Combined Long Multiply & Add/Sub
9408
9409 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9410 match(Set dst (AddL src3 (MulL src1 src2)));
9411
9412 ins_cost(INSN_COST * 5);
9413 format %{ "madd $dst, $src1, $src2, $src3" %}
9414
9415 ins_encode %{
9416 __ madd(as_Register($dst$$reg),
9417 as_Register($src1$$reg),
9418 as_Register($src2$$reg),
9419 as_Register($src3$$reg));
9420 %}
9421
9422 ins_pipe(lmac_reg_reg);
9423 %}
9424
9425 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9426 match(Set dst (SubL src3 (MulL src1 src2)));
9427
9428 ins_cost(INSN_COST * 5);
9429 format %{ "msub $dst, $src1, $src2, $src3" %}
9430
9431 ins_encode %{
9432 __ msub(as_Register($dst$$reg),
9433 as_Register($src1$$reg),
9434 as_Register($src2$$reg),
9435 as_Register($src3$$reg));
9436 %}
9437
9438 ins_pipe(lmac_reg_reg);
9439 %}
9440
9441 // Combined Long Multiply & Neg
9442
9443 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9444 match(Set dst (MulL (SubL zero src1) src2));
9445
9446 ins_cost(INSN_COST * 5);
9447 format %{ "mneg $dst, $src1, $src2" %}
9448
9449 ins_encode %{
9450 __ mneg(as_Register($dst$$reg),
9451 as_Register($src1$$reg),
9452 as_Register($src2$$reg));
9453 %}
9454
9455 ins_pipe(lmac_reg_reg);
9456 %}
9457
9458 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9459
9460 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9461 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9462
9463 ins_cost(INSN_COST * 3);
9464 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9465
9466 ins_encode %{
9467 __ smaddl(as_Register($dst$$reg),
9468 as_Register($src1$$reg),
9469 as_Register($src2$$reg),
9470 as_Register($src3$$reg));
9471 %}
9472
9473 ins_pipe(imac_reg_reg);
9474 %}
9475
9476 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9477 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9478
9479 ins_cost(INSN_COST * 3);
9480 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9481
9482 ins_encode %{
9483 __ smsubl(as_Register($dst$$reg),
9484 as_Register($src1$$reg),
9485 as_Register($src2$$reg),
9486 as_Register($src3$$reg));
9487 %}
9488
9489 ins_pipe(imac_reg_reg);
9490 %}
9491
9492 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9493 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9494
9495 ins_cost(INSN_COST * 3);
9496 format %{ "smnegl $dst, $src1, $src2" %}
9497
9498 ins_encode %{
9499 __ smnegl(as_Register($dst$$reg),
9500 as_Register($src1$$reg),
9501 as_Register($src2$$reg));
9502 %}
9503
9504 ins_pipe(imac_reg_reg);
9505 %}
9506
9507 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9508
9509 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9510 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9511
9512 ins_cost(INSN_COST * 5);
9513 format %{ "mulw rscratch1, $src1, $src2\n\t"
9514 "maddw $dst, $src3, $src4, rscratch1" %}
9515
9516 ins_encode %{
9517 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9518 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9519
9520 ins_pipe(imac_reg_reg);
9521 %}
9522
9523 // Integer Divide
9524
9525 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9526 match(Set dst (DivI src1 src2));
9527
9528 ins_cost(INSN_COST * 19);
9529 format %{ "sdivw $dst, $src1, $src2" %}
9530
9531 ins_encode(aarch64_enc_divw(dst, src1, src2));
9532 ins_pipe(idiv_reg_reg);
9533 %}
9534
9535 // Long Divide
9536
9537 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9538 match(Set dst (DivL src1 src2));
9539
9540 ins_cost(INSN_COST * 35);
9541 format %{ "sdiv $dst, $src1, $src2" %}
9542
9543 ins_encode(aarch64_enc_div(dst, src1, src2));
9544 ins_pipe(ldiv_reg_reg);
9545 %}
9546
9547 // Integer Remainder
9548
9549 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9550 match(Set dst (ModI src1 src2));
9551
9552 ins_cost(INSN_COST * 22);
9553 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9554 "msubw $dst, rscratch1, $src2, $src1" %}
9555
9556 ins_encode(aarch64_enc_modw(dst, src1, src2));
9557 ins_pipe(idiv_reg_reg);
9558 %}
9559
9560 // Long Remainder
9561
9562 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9563 match(Set dst (ModL src1 src2));
9564
9565 ins_cost(INSN_COST * 38);
9566 format %{ "sdiv rscratch1, $src1, $src2\n"
9567 "msub $dst, rscratch1, $src2, $src1" %}
9568
9569 ins_encode(aarch64_enc_mod(dst, src1, src2));
9570 ins_pipe(ldiv_reg_reg);
9571 %}
9572
9573 // Unsigned Integer Divide
9574
9575 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9576 match(Set dst (UDivI src1 src2));
9577
9578 ins_cost(INSN_COST * 19);
9579 format %{ "udivw $dst, $src1, $src2" %}
9580
9581 ins_encode %{
9582 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9583 %}
9584
9585 ins_pipe(idiv_reg_reg);
9586 %}
9587
9588 // Unsigned Long Divide
9589
9590 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9591 match(Set dst (UDivL src1 src2));
9592
9593 ins_cost(INSN_COST * 35);
9594 format %{ "udiv $dst, $src1, $src2" %}
9595
9596 ins_encode %{
9597 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9598 %}
9599
9600 ins_pipe(ldiv_reg_reg);
9601 %}
9602
9603 // Unsigned Integer Remainder
9604
9605 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9606 match(Set dst (UModI src1 src2));
9607
9608 ins_cost(INSN_COST * 22);
9609 format %{ "udivw rscratch1, $src1, $src2\n\t"
9610 "msubw $dst, rscratch1, $src2, $src1" %}
9611
9612 ins_encode %{
9613 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9614 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9615 %}
9616
9617 ins_pipe(idiv_reg_reg);
9618 %}
9619
9620 // Unsigned Long Remainder
9621
9622 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9623 match(Set dst (UModL src1 src2));
9624
9625 ins_cost(INSN_COST * 38);
9626 format %{ "udiv rscratch1, $src1, $src2\n"
9627 "msub $dst, rscratch1, $src2, $src1" %}
9628
9629 ins_encode %{
9630 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9631 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9632 %}
9633
9634 ins_pipe(ldiv_reg_reg);
9635 %}
9636
9637 // Integer Shifts
9638
9639 // Shift Left Register
9640 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9641 match(Set dst (LShiftI src1 src2));
9642
9643 ins_cost(INSN_COST * 2);
9644 format %{ "lslvw $dst, $src1, $src2" %}
9645
9646 ins_encode %{
9647 __ lslvw(as_Register($dst$$reg),
9648 as_Register($src1$$reg),
9649 as_Register($src2$$reg));
9650 %}
9651
9652 ins_pipe(ialu_reg_reg_vshift);
9653 %}
9654
9655 // Shift Left Immediate
9656 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9657 match(Set dst (LShiftI src1 src2));
9658
9659 ins_cost(INSN_COST);
9660 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9661
9662 ins_encode %{
9663 __ lslw(as_Register($dst$$reg),
9664 as_Register($src1$$reg),
9665 $src2$$constant & 0x1f);
9666 %}
9667
9668 ins_pipe(ialu_reg_shift);
9669 %}
9670
9671 // Shift Right Logical Register
9672 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9673 match(Set dst (URShiftI src1 src2));
9674
9675 ins_cost(INSN_COST * 2);
9676 format %{ "lsrvw $dst, $src1, $src2" %}
9677
9678 ins_encode %{
9679 __ lsrvw(as_Register($dst$$reg),
9680 as_Register($src1$$reg),
9681 as_Register($src2$$reg));
9682 %}
9683
9684 ins_pipe(ialu_reg_reg_vshift);
9685 %}
9686
9687 // Shift Right Logical Immediate
9688 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9689 match(Set dst (URShiftI src1 src2));
9690
9691 ins_cost(INSN_COST);
9692 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9693
9694 ins_encode %{
9695 __ lsrw(as_Register($dst$$reg),
9696 as_Register($src1$$reg),
9697 $src2$$constant & 0x1f);
9698 %}
9699
9700 ins_pipe(ialu_reg_shift);
9701 %}
9702
9703 // Shift Right Arithmetic Register
9704 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9705 match(Set dst (RShiftI src1 src2));
9706
9707 ins_cost(INSN_COST * 2);
9708 format %{ "asrvw $dst, $src1, $src2" %}
9709
9710 ins_encode %{
9711 __ asrvw(as_Register($dst$$reg),
9712 as_Register($src1$$reg),
9713 as_Register($src2$$reg));
9714 %}
9715
9716 ins_pipe(ialu_reg_reg_vshift);
9717 %}
9718
9719 // Shift Right Arithmetic Immediate
9720 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9721 match(Set dst (RShiftI src1 src2));
9722
9723 ins_cost(INSN_COST);
9724 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9725
9726 ins_encode %{
9727 __ asrw(as_Register($dst$$reg),
9728 as_Register($src1$$reg),
9729 $src2$$constant & 0x1f);
9730 %}
9731
9732 ins_pipe(ialu_reg_shift);
9733 %}
9734
9735 // Combined Int Mask and Right Shift (using UBFM)
9736 // TODO
9737
9738 // Long Shifts
9739
9740 // Shift Left Register
9741 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9742 match(Set dst (LShiftL src1 src2));
9743
9744 ins_cost(INSN_COST * 2);
9745 format %{ "lslv $dst, $src1, $src2" %}
9746
9747 ins_encode %{
9748 __ lslv(as_Register($dst$$reg),
9749 as_Register($src1$$reg),
9750 as_Register($src2$$reg));
9751 %}
9752
9753 ins_pipe(ialu_reg_reg_vshift);
9754 %}
9755
9756 // Shift Left Immediate
9757 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9758 match(Set dst (LShiftL src1 src2));
9759
9760 ins_cost(INSN_COST);
9761 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9762
9763 ins_encode %{
9764 __ lsl(as_Register($dst$$reg),
9765 as_Register($src1$$reg),
9766 $src2$$constant & 0x3f);
9767 %}
9768
9769 ins_pipe(ialu_reg_shift);
9770 %}
9771
9772 // Shift Right Logical Register
9773 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9774 match(Set dst (URShiftL src1 src2));
9775
9776 ins_cost(INSN_COST * 2);
9777 format %{ "lsrv $dst, $src1, $src2" %}
9778
9779 ins_encode %{
9780 __ lsrv(as_Register($dst$$reg),
9781 as_Register($src1$$reg),
9782 as_Register($src2$$reg));
9783 %}
9784
9785 ins_pipe(ialu_reg_reg_vshift);
9786 %}
9787
9788 // Shift Right Logical Immediate
9789 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9790 match(Set dst (URShiftL src1 src2));
9791
9792 ins_cost(INSN_COST);
9793 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9794
9795 ins_encode %{
9796 __ lsr(as_Register($dst$$reg),
9797 as_Register($src1$$reg),
9798 $src2$$constant & 0x3f);
9799 %}
9800
9801 ins_pipe(ialu_reg_shift);
9802 %}
9803
9804 // A special-case pattern for card table stores.
9805 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9806 match(Set dst (URShiftL (CastP2X src1) src2));
9807
9808 ins_cost(INSN_COST);
9809 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9810
9811 ins_encode %{
9812 __ lsr(as_Register($dst$$reg),
9813 as_Register($src1$$reg),
9814 $src2$$constant & 0x3f);
9815 %}
9816
9817 ins_pipe(ialu_reg_shift);
9818 %}
9819
9820 // Shift Right Arithmetic Register
9821 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9822 match(Set dst (RShiftL src1 src2));
9823
9824 ins_cost(INSN_COST * 2);
9825 format %{ "asrv $dst, $src1, $src2" %}
9826
9827 ins_encode %{
9828 __ asrv(as_Register($dst$$reg),
9829 as_Register($src1$$reg),
9830 as_Register($src2$$reg));
9831 %}
9832
9833 ins_pipe(ialu_reg_reg_vshift);
9834 %}
9835
9836 // Shift Right Arithmetic Immediate
9837 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9838 match(Set dst (RShiftL src1 src2));
9839
9840 ins_cost(INSN_COST);
9841 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9842
9843 ins_encode %{
9844 __ asr(as_Register($dst$$reg),
9845 as_Register($src1$$reg),
9846 $src2$$constant & 0x3f);
9847 %}
9848
9849 ins_pipe(ialu_reg_shift);
9850 %}
9851
9852 // BEGIN This section of the file is automatically generated. Do not edit --------------
9853 // This section is generated from aarch64_ad.m4
9854
9855 // This pattern is automatically generated from aarch64_ad.m4
9856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9857 instruct regL_not_reg(iRegLNoSp dst,
9858 iRegL src1, immL_M1 m1,
9859 rFlagsReg cr) %{
9860 match(Set dst (XorL src1 m1));
9861 ins_cost(INSN_COST);
9862 format %{ "eon $dst, $src1, zr" %}
9863
9864 ins_encode %{
9865 __ eon(as_Register($dst$$reg),
9866 as_Register($src1$$reg),
9867 zr,
9868 Assembler::LSL, 0);
9869 %}
9870
9871 ins_pipe(ialu_reg);
9872 %}
9873
9874 // This pattern is automatically generated from aarch64_ad.m4
9875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9876 instruct regI_not_reg(iRegINoSp dst,
9877 iRegIorL2I src1, immI_M1 m1,
9878 rFlagsReg cr) %{
9879 match(Set dst (XorI src1 m1));
9880 ins_cost(INSN_COST);
9881 format %{ "eonw $dst, $src1, zr" %}
9882
9883 ins_encode %{
9884 __ eonw(as_Register($dst$$reg),
9885 as_Register($src1$$reg),
9886 zr,
9887 Assembler::LSL, 0);
9888 %}
9889
9890 ins_pipe(ialu_reg);
9891 %}
9892
9893 // This pattern is automatically generated from aarch64_ad.m4
9894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9895 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9896 immI0 zero, iRegIorL2I src1, immI src2) %{
9897 match(Set dst (SubI zero (URShiftI src1 src2)));
9898
9899 ins_cost(1.9 * INSN_COST);
9900 format %{ "negw $dst, $src1, LSR $src2" %}
9901
9902 ins_encode %{
9903 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9904 Assembler::LSR, $src2$$constant & 0x1f);
9905 %}
9906
9907 ins_pipe(ialu_reg_shift);
9908 %}
9909
9910 // This pattern is automatically generated from aarch64_ad.m4
9911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9912 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9913 immI0 zero, iRegIorL2I src1, immI src2) %{
9914 match(Set dst (SubI zero (RShiftI src1 src2)));
9915
9916 ins_cost(1.9 * INSN_COST);
9917 format %{ "negw $dst, $src1, ASR $src2" %}
9918
9919 ins_encode %{
9920 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9921 Assembler::ASR, $src2$$constant & 0x1f);
9922 %}
9923
9924 ins_pipe(ialu_reg_shift);
9925 %}
9926
9927 // This pattern is automatically generated from aarch64_ad.m4
9928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9929 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9930 immI0 zero, iRegIorL2I src1, immI src2) %{
9931 match(Set dst (SubI zero (LShiftI src1 src2)));
9932
9933 ins_cost(1.9 * INSN_COST);
9934 format %{ "negw $dst, $src1, LSL $src2" %}
9935
9936 ins_encode %{
9937 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9938 Assembler::LSL, $src2$$constant & 0x1f);
9939 %}
9940
9941 ins_pipe(ialu_reg_shift);
9942 %}
9943
9944 // This pattern is automatically generated from aarch64_ad.m4
9945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9946 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9947 immL0 zero, iRegL src1, immI src2) %{
9948 match(Set dst (SubL zero (URShiftL src1 src2)));
9949
9950 ins_cost(1.9 * INSN_COST);
9951 format %{ "neg $dst, $src1, LSR $src2" %}
9952
9953 ins_encode %{
9954 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9955 Assembler::LSR, $src2$$constant & 0x3f);
9956 %}
9957
9958 ins_pipe(ialu_reg_shift);
9959 %}
9960
9961 // This pattern is automatically generated from aarch64_ad.m4
9962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9963 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9964 immL0 zero, iRegL src1, immI src2) %{
9965 match(Set dst (SubL zero (RShiftL src1 src2)));
9966
9967 ins_cost(1.9 * INSN_COST);
9968 format %{ "neg $dst, $src1, ASR $src2" %}
9969
9970 ins_encode %{
9971 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9972 Assembler::ASR, $src2$$constant & 0x3f);
9973 %}
9974
9975 ins_pipe(ialu_reg_shift);
9976 %}
9977
9978 // This pattern is automatically generated from aarch64_ad.m4
9979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9980 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9981 immL0 zero, iRegL src1, immI src2) %{
9982 match(Set dst (SubL zero (LShiftL src1 src2)));
9983
9984 ins_cost(1.9 * INSN_COST);
9985 format %{ "neg $dst, $src1, LSL $src2" %}
9986
9987 ins_encode %{
9988 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9989 Assembler::LSL, $src2$$constant & 0x3f);
9990 %}
9991
9992 ins_pipe(ialu_reg_shift);
9993 %}
9994
9995 // This pattern is automatically generated from aarch64_ad.m4
9996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9997 instruct AndI_reg_not_reg(iRegINoSp dst,
9998 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
9999 match(Set dst (AndI src1 (XorI src2 m1)));
10000 ins_cost(INSN_COST);
10001 format %{ "bicw $dst, $src1, $src2" %}
10002
10003 ins_encode %{
10004 __ bicw(as_Register($dst$$reg),
10005 as_Register($src1$$reg),
10006 as_Register($src2$$reg),
10007 Assembler::LSL, 0);
10008 %}
10009
10010 ins_pipe(ialu_reg_reg);
10011 %}
10012
10013 // This pattern is automatically generated from aarch64_ad.m4
10014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10015 instruct AndL_reg_not_reg(iRegLNoSp dst,
10016 iRegL src1, iRegL src2, immL_M1 m1) %{
10017 match(Set dst (AndL src1 (XorL src2 m1)));
10018 ins_cost(INSN_COST);
10019 format %{ "bic $dst, $src1, $src2" %}
10020
10021 ins_encode %{
10022 __ bic(as_Register($dst$$reg),
10023 as_Register($src1$$reg),
10024 as_Register($src2$$reg),
10025 Assembler::LSL, 0);
10026 %}
10027
10028 ins_pipe(ialu_reg_reg);
10029 %}
10030
10031 // This pattern is automatically generated from aarch64_ad.m4
10032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10033 instruct OrI_reg_not_reg(iRegINoSp dst,
10034 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10035 match(Set dst (OrI src1 (XorI src2 m1)));
10036 ins_cost(INSN_COST);
10037 format %{ "ornw $dst, $src1, $src2" %}
10038
10039 ins_encode %{
10040 __ ornw(as_Register($dst$$reg),
10041 as_Register($src1$$reg),
10042 as_Register($src2$$reg),
10043 Assembler::LSL, 0);
10044 %}
10045
10046 ins_pipe(ialu_reg_reg);
10047 %}
10048
10049 // This pattern is automatically generated from aarch64_ad.m4
10050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10051 instruct OrL_reg_not_reg(iRegLNoSp dst,
10052 iRegL src1, iRegL src2, immL_M1 m1) %{
10053 match(Set dst (OrL src1 (XorL src2 m1)));
10054 ins_cost(INSN_COST);
10055 format %{ "orn $dst, $src1, $src2" %}
10056
10057 ins_encode %{
10058 __ orn(as_Register($dst$$reg),
10059 as_Register($src1$$reg),
10060 as_Register($src2$$reg),
10061 Assembler::LSL, 0);
10062 %}
10063
10064 ins_pipe(ialu_reg_reg);
10065 %}
10066
10067 // This pattern is automatically generated from aarch64_ad.m4
10068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10069 instruct XorI_reg_not_reg(iRegINoSp dst,
10070 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10071 match(Set dst (XorI m1 (XorI src2 src1)));
10072 ins_cost(INSN_COST);
10073 format %{ "eonw $dst, $src1, $src2" %}
10074
10075 ins_encode %{
10076 __ eonw(as_Register($dst$$reg),
10077 as_Register($src1$$reg),
10078 as_Register($src2$$reg),
10079 Assembler::LSL, 0);
10080 %}
10081
10082 ins_pipe(ialu_reg_reg);
10083 %}
10084
10085 // This pattern is automatically generated from aarch64_ad.m4
10086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10087 instruct XorL_reg_not_reg(iRegLNoSp dst,
10088 iRegL src1, iRegL src2, immL_M1 m1) %{
10089 match(Set dst (XorL m1 (XorL src2 src1)));
10090 ins_cost(INSN_COST);
10091 format %{ "eon $dst, $src1, $src2" %}
10092
10093 ins_encode %{
10094 __ eon(as_Register($dst$$reg),
10095 as_Register($src1$$reg),
10096 as_Register($src2$$reg),
10097 Assembler::LSL, 0);
10098 %}
10099
10100 ins_pipe(ialu_reg_reg);
10101 %}
10102
10103 // This pattern is automatically generated from aarch64_ad.m4
10104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10105 // val & (-1 ^ (val >>> shift)) ==> bicw
10106 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10107 iRegIorL2I src1, iRegIorL2I src2,
10108 immI src3, immI_M1 src4) %{
10109 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10110 ins_cost(1.9 * INSN_COST);
10111 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10112
10113 ins_encode %{
10114 __ bicw(as_Register($dst$$reg),
10115 as_Register($src1$$reg),
10116 as_Register($src2$$reg),
10117 Assembler::LSR,
10118 $src3$$constant & 0x1f);
10119 %}
10120
10121 ins_pipe(ialu_reg_reg_shift);
10122 %}
10123
10124 // This pattern is automatically generated from aarch64_ad.m4
10125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10126 // val & (-1 ^ (val >>> shift)) ==> bic
10127 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10128 iRegL src1, iRegL src2,
10129 immI src3, immL_M1 src4) %{
10130 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10131 ins_cost(1.9 * INSN_COST);
10132 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10133
10134 ins_encode %{
10135 __ bic(as_Register($dst$$reg),
10136 as_Register($src1$$reg),
10137 as_Register($src2$$reg),
10138 Assembler::LSR,
10139 $src3$$constant & 0x3f);
10140 %}
10141
10142 ins_pipe(ialu_reg_reg_shift);
10143 %}
10144
10145 // This pattern is automatically generated from aarch64_ad.m4
10146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10147 // val & (-1 ^ (val >> shift)) ==> bicw
10148 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10149 iRegIorL2I src1, iRegIorL2I src2,
10150 immI src3, immI_M1 src4) %{
10151 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10152 ins_cost(1.9 * INSN_COST);
10153 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10154
10155 ins_encode %{
10156 __ bicw(as_Register($dst$$reg),
10157 as_Register($src1$$reg),
10158 as_Register($src2$$reg),
10159 Assembler::ASR,
10160 $src3$$constant & 0x1f);
10161 %}
10162
10163 ins_pipe(ialu_reg_reg_shift);
10164 %}
10165
10166 // This pattern is automatically generated from aarch64_ad.m4
10167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10168 // val & (-1 ^ (val >> shift)) ==> bic
10169 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10170 iRegL src1, iRegL src2,
10171 immI src3, immL_M1 src4) %{
10172 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10173 ins_cost(1.9 * INSN_COST);
10174 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10175
10176 ins_encode %{
10177 __ bic(as_Register($dst$$reg),
10178 as_Register($src1$$reg),
10179 as_Register($src2$$reg),
10180 Assembler::ASR,
10181 $src3$$constant & 0x3f);
10182 %}
10183
10184 ins_pipe(ialu_reg_reg_shift);
10185 %}
10186
10187 // This pattern is automatically generated from aarch64_ad.m4
10188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10189 // val & (-1 ^ (val ror shift)) ==> bicw
10190 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10191 iRegIorL2I src1, iRegIorL2I src2,
10192 immI src3, immI_M1 src4) %{
10193 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10194 ins_cost(1.9 * INSN_COST);
10195 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10196
10197 ins_encode %{
10198 __ bicw(as_Register($dst$$reg),
10199 as_Register($src1$$reg),
10200 as_Register($src2$$reg),
10201 Assembler::ROR,
10202 $src3$$constant & 0x1f);
10203 %}
10204
10205 ins_pipe(ialu_reg_reg_shift);
10206 %}
10207
10208 // This pattern is automatically generated from aarch64_ad.m4
10209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10210 // val & (-1 ^ (val ror shift)) ==> bic
10211 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10212 iRegL src1, iRegL src2,
10213 immI src3, immL_M1 src4) %{
10214 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10215 ins_cost(1.9 * INSN_COST);
10216 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10217
10218 ins_encode %{
10219 __ bic(as_Register($dst$$reg),
10220 as_Register($src1$$reg),
10221 as_Register($src2$$reg),
10222 Assembler::ROR,
10223 $src3$$constant & 0x3f);
10224 %}
10225
10226 ins_pipe(ialu_reg_reg_shift);
10227 %}
10228
10229 // This pattern is automatically generated from aarch64_ad.m4
10230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10231 // val & (-1 ^ (val << shift)) ==> bicw
10232 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10233 iRegIorL2I src1, iRegIorL2I src2,
10234 immI src3, immI_M1 src4) %{
10235 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10236 ins_cost(1.9 * INSN_COST);
10237 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10238
10239 ins_encode %{
10240 __ bicw(as_Register($dst$$reg),
10241 as_Register($src1$$reg),
10242 as_Register($src2$$reg),
10243 Assembler::LSL,
10244 $src3$$constant & 0x1f);
10245 %}
10246
10247 ins_pipe(ialu_reg_reg_shift);
10248 %}
10249
10250 // This pattern is automatically generated from aarch64_ad.m4
10251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10252 // val & (-1 ^ (val << shift)) ==> bic
10253 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10254 iRegL src1, iRegL src2,
10255 immI src3, immL_M1 src4) %{
10256 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10257 ins_cost(1.9 * INSN_COST);
10258 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10259
10260 ins_encode %{
10261 __ bic(as_Register($dst$$reg),
10262 as_Register($src1$$reg),
10263 as_Register($src2$$reg),
10264 Assembler::LSL,
10265 $src3$$constant & 0x3f);
10266 %}
10267
10268 ins_pipe(ialu_reg_reg_shift);
10269 %}
10270
10271 // This pattern is automatically generated from aarch64_ad.m4
10272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10273 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10274 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10275 iRegIorL2I src1, iRegIorL2I src2,
10276 immI src3, immI_M1 src4) %{
10277 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10278 ins_cost(1.9 * INSN_COST);
10279 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10280
10281 ins_encode %{
10282 __ eonw(as_Register($dst$$reg),
10283 as_Register($src1$$reg),
10284 as_Register($src2$$reg),
10285 Assembler::LSR,
10286 $src3$$constant & 0x1f);
10287 %}
10288
10289 ins_pipe(ialu_reg_reg_shift);
10290 %}
10291
10292 // This pattern is automatically generated from aarch64_ad.m4
10293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10294 // val ^ (-1 ^ (val >>> shift)) ==> eon
10295 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10296 iRegL src1, iRegL src2,
10297 immI src3, immL_M1 src4) %{
10298 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10299 ins_cost(1.9 * INSN_COST);
10300 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10301
10302 ins_encode %{
10303 __ eon(as_Register($dst$$reg),
10304 as_Register($src1$$reg),
10305 as_Register($src2$$reg),
10306 Assembler::LSR,
10307 $src3$$constant & 0x3f);
10308 %}
10309
10310 ins_pipe(ialu_reg_reg_shift);
10311 %}
10312
10313 // This pattern is automatically generated from aarch64_ad.m4
10314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10315 // val ^ (-1 ^ (val >> shift)) ==> eonw
10316 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10317 iRegIorL2I src1, iRegIorL2I src2,
10318 immI src3, immI_M1 src4) %{
10319 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10320 ins_cost(1.9 * INSN_COST);
10321 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10322
10323 ins_encode %{
10324 __ eonw(as_Register($dst$$reg),
10325 as_Register($src1$$reg),
10326 as_Register($src2$$reg),
10327 Assembler::ASR,
10328 $src3$$constant & 0x1f);
10329 %}
10330
10331 ins_pipe(ialu_reg_reg_shift);
10332 %}
10333
10334 // This pattern is automatically generated from aarch64_ad.m4
10335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10336 // val ^ (-1 ^ (val >> shift)) ==> eon
10337 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10338 iRegL src1, iRegL src2,
10339 immI src3, immL_M1 src4) %{
10340 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10341 ins_cost(1.9 * INSN_COST);
10342 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10343
10344 ins_encode %{
10345 __ eon(as_Register($dst$$reg),
10346 as_Register($src1$$reg),
10347 as_Register($src2$$reg),
10348 Assembler::ASR,
10349 $src3$$constant & 0x3f);
10350 %}
10351
10352 ins_pipe(ialu_reg_reg_shift);
10353 %}
10354
10355 // This pattern is automatically generated from aarch64_ad.m4
10356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10357 // val ^ (-1 ^ (val ror shift)) ==> eonw
10358 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10359 iRegIorL2I src1, iRegIorL2I src2,
10360 immI src3, immI_M1 src4) %{
10361 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10362 ins_cost(1.9 * INSN_COST);
10363 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10364
10365 ins_encode %{
10366 __ eonw(as_Register($dst$$reg),
10367 as_Register($src1$$reg),
10368 as_Register($src2$$reg),
10369 Assembler::ROR,
10370 $src3$$constant & 0x1f);
10371 %}
10372
10373 ins_pipe(ialu_reg_reg_shift);
10374 %}
10375
10376 // This pattern is automatically generated from aarch64_ad.m4
10377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10378 // val ^ (-1 ^ (val ror shift)) ==> eon
10379 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10380 iRegL src1, iRegL src2,
10381 immI src3, immL_M1 src4) %{
10382 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10383 ins_cost(1.9 * INSN_COST);
10384 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10385
10386 ins_encode %{
10387 __ eon(as_Register($dst$$reg),
10388 as_Register($src1$$reg),
10389 as_Register($src2$$reg),
10390 Assembler::ROR,
10391 $src3$$constant & 0x3f);
10392 %}
10393
10394 ins_pipe(ialu_reg_reg_shift);
10395 %}
10396
10397 // This pattern is automatically generated from aarch64_ad.m4
10398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10399 // val ^ (-1 ^ (val << shift)) ==> eonw
10400 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10401 iRegIorL2I src1, iRegIorL2I src2,
10402 immI src3, immI_M1 src4) %{
10403 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10404 ins_cost(1.9 * INSN_COST);
10405 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10406
10407 ins_encode %{
10408 __ eonw(as_Register($dst$$reg),
10409 as_Register($src1$$reg),
10410 as_Register($src2$$reg),
10411 Assembler::LSL,
10412 $src3$$constant & 0x1f);
10413 %}
10414
10415 ins_pipe(ialu_reg_reg_shift);
10416 %}
10417
10418 // This pattern is automatically generated from aarch64_ad.m4
10419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10420 // val ^ (-1 ^ (val << shift)) ==> eon
10421 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10422 iRegL src1, iRegL src2,
10423 immI src3, immL_M1 src4) %{
10424 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10425 ins_cost(1.9 * INSN_COST);
10426 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10427
10428 ins_encode %{
10429 __ eon(as_Register($dst$$reg),
10430 as_Register($src1$$reg),
10431 as_Register($src2$$reg),
10432 Assembler::LSL,
10433 $src3$$constant & 0x3f);
10434 %}
10435
10436 ins_pipe(ialu_reg_reg_shift);
10437 %}
10438
10439 // This pattern is automatically generated from aarch64_ad.m4
10440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10441 // val | (-1 ^ (val >>> shift)) ==> ornw
10442 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10443 iRegIorL2I src1, iRegIorL2I src2,
10444 immI src3, immI_M1 src4) %{
10445 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10446 ins_cost(1.9 * INSN_COST);
10447 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10448
10449 ins_encode %{
10450 __ ornw(as_Register($dst$$reg),
10451 as_Register($src1$$reg),
10452 as_Register($src2$$reg),
10453 Assembler::LSR,
10454 $src3$$constant & 0x1f);
10455 %}
10456
10457 ins_pipe(ialu_reg_reg_shift);
10458 %}
10459
10460 // This pattern is automatically generated from aarch64_ad.m4
10461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10462 // val | (-1 ^ (val >>> shift)) ==> orn
10463 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10464 iRegL src1, iRegL src2,
10465 immI src3, immL_M1 src4) %{
10466 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10467 ins_cost(1.9 * INSN_COST);
10468 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10469
10470 ins_encode %{
10471 __ orn(as_Register($dst$$reg),
10472 as_Register($src1$$reg),
10473 as_Register($src2$$reg),
10474 Assembler::LSR,
10475 $src3$$constant & 0x3f);
10476 %}
10477
10478 ins_pipe(ialu_reg_reg_shift);
10479 %}
10480
10481 // This pattern is automatically generated from aarch64_ad.m4
10482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10483 // val | (-1 ^ (val >> shift)) ==> ornw
10484 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10485 iRegIorL2I src1, iRegIorL2I src2,
10486 immI src3, immI_M1 src4) %{
10487 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10488 ins_cost(1.9 * INSN_COST);
10489 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10490
10491 ins_encode %{
10492 __ ornw(as_Register($dst$$reg),
10493 as_Register($src1$$reg),
10494 as_Register($src2$$reg),
10495 Assembler::ASR,
10496 $src3$$constant & 0x1f);
10497 %}
10498
10499 ins_pipe(ialu_reg_reg_shift);
10500 %}
10501
10502 // This pattern is automatically generated from aarch64_ad.m4
10503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10504 // val | (-1 ^ (val >> shift)) ==> orn
10505 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10506 iRegL src1, iRegL src2,
10507 immI src3, immL_M1 src4) %{
10508 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10509 ins_cost(1.9 * INSN_COST);
10510 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10511
10512 ins_encode %{
10513 __ orn(as_Register($dst$$reg),
10514 as_Register($src1$$reg),
10515 as_Register($src2$$reg),
10516 Assembler::ASR,
10517 $src3$$constant & 0x3f);
10518 %}
10519
10520 ins_pipe(ialu_reg_reg_shift);
10521 %}
10522
10523 // This pattern is automatically generated from aarch64_ad.m4
10524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10525 // val | (-1 ^ (val ror shift)) ==> ornw
10526 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10527 iRegIorL2I src1, iRegIorL2I src2,
10528 immI src3, immI_M1 src4) %{
10529 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10530 ins_cost(1.9 * INSN_COST);
10531 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10532
10533 ins_encode %{
10534 __ ornw(as_Register($dst$$reg),
10535 as_Register($src1$$reg),
10536 as_Register($src2$$reg),
10537 Assembler::ROR,
10538 $src3$$constant & 0x1f);
10539 %}
10540
10541 ins_pipe(ialu_reg_reg_shift);
10542 %}
10543
10544 // This pattern is automatically generated from aarch64_ad.m4
10545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10546 // val | (-1 ^ (val ror shift)) ==> orn
10547 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10548 iRegL src1, iRegL src2,
10549 immI src3, immL_M1 src4) %{
10550 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10551 ins_cost(1.9 * INSN_COST);
10552 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10553
10554 ins_encode %{
10555 __ orn(as_Register($dst$$reg),
10556 as_Register($src1$$reg),
10557 as_Register($src2$$reg),
10558 Assembler::ROR,
10559 $src3$$constant & 0x3f);
10560 %}
10561
10562 ins_pipe(ialu_reg_reg_shift);
10563 %}
10564
10565 // This pattern is automatically generated from aarch64_ad.m4
10566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10567 // val | (-1 ^ (val << shift)) ==> ornw
10568 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10569 iRegIorL2I src1, iRegIorL2I src2,
10570 immI src3, immI_M1 src4) %{
10571 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10572 ins_cost(1.9 * INSN_COST);
10573 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10574
10575 ins_encode %{
10576 __ ornw(as_Register($dst$$reg),
10577 as_Register($src1$$reg),
10578 as_Register($src2$$reg),
10579 Assembler::LSL,
10580 $src3$$constant & 0x1f);
10581 %}
10582
10583 ins_pipe(ialu_reg_reg_shift);
10584 %}
10585
10586 // This pattern is automatically generated from aarch64_ad.m4
10587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10588 // val | (-1 ^ (val << shift)) ==> orn
10589 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10590 iRegL src1, iRegL src2,
10591 immI src3, immL_M1 src4) %{
10592 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10593 ins_cost(1.9 * INSN_COST);
10594 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10595
10596 ins_encode %{
10597 __ orn(as_Register($dst$$reg),
10598 as_Register($src1$$reg),
10599 as_Register($src2$$reg),
10600 Assembler::LSL,
10601 $src3$$constant & 0x3f);
10602 %}
10603
10604 ins_pipe(ialu_reg_reg_shift);
10605 %}
10606
10607 // This pattern is automatically generated from aarch64_ad.m4
10608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10609 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10610 iRegIorL2I src1, iRegIorL2I src2,
10611 immI src3) %{
10612 match(Set dst (AndI src1 (URShiftI src2 src3)));
10613
10614 ins_cost(1.9 * INSN_COST);
10615 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10616
10617 ins_encode %{
10618 __ andw(as_Register($dst$$reg),
10619 as_Register($src1$$reg),
10620 as_Register($src2$$reg),
10621 Assembler::LSR,
10622 $src3$$constant & 0x1f);
10623 %}
10624
10625 ins_pipe(ialu_reg_reg_shift);
10626 %}
10627
10628 // This pattern is automatically generated from aarch64_ad.m4
10629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10630 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10631 iRegL src1, iRegL src2,
10632 immI src3) %{
10633 match(Set dst (AndL src1 (URShiftL src2 src3)));
10634
10635 ins_cost(1.9 * INSN_COST);
10636 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10637
10638 ins_encode %{
10639 __ andr(as_Register($dst$$reg),
10640 as_Register($src1$$reg),
10641 as_Register($src2$$reg),
10642 Assembler::LSR,
10643 $src3$$constant & 0x3f);
10644 %}
10645
10646 ins_pipe(ialu_reg_reg_shift);
10647 %}
10648
10649 // This pattern is automatically generated from aarch64_ad.m4
10650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10651 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10652 iRegIorL2I src1, iRegIorL2I src2,
10653 immI src3) %{
10654 match(Set dst (AndI src1 (RShiftI src2 src3)));
10655
10656 ins_cost(1.9 * INSN_COST);
10657 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10658
10659 ins_encode %{
10660 __ andw(as_Register($dst$$reg),
10661 as_Register($src1$$reg),
10662 as_Register($src2$$reg),
10663 Assembler::ASR,
10664 $src3$$constant & 0x1f);
10665 %}
10666
10667 ins_pipe(ialu_reg_reg_shift);
10668 %}
10669
10670 // This pattern is automatically generated from aarch64_ad.m4
10671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10672 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10673 iRegL src1, iRegL src2,
10674 immI src3) %{
10675 match(Set dst (AndL src1 (RShiftL src2 src3)));
10676
10677 ins_cost(1.9 * INSN_COST);
10678 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10679
10680 ins_encode %{
10681 __ andr(as_Register($dst$$reg),
10682 as_Register($src1$$reg),
10683 as_Register($src2$$reg),
10684 Assembler::ASR,
10685 $src3$$constant & 0x3f);
10686 %}
10687
10688 ins_pipe(ialu_reg_reg_shift);
10689 %}
10690
10691 // This pattern is automatically generated from aarch64_ad.m4
10692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10693 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10694 iRegIorL2I src1, iRegIorL2I src2,
10695 immI src3) %{
10696 match(Set dst (AndI src1 (LShiftI src2 src3)));
10697
10698 ins_cost(1.9 * INSN_COST);
10699 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10700
10701 ins_encode %{
10702 __ andw(as_Register($dst$$reg),
10703 as_Register($src1$$reg),
10704 as_Register($src2$$reg),
10705 Assembler::LSL,
10706 $src3$$constant & 0x1f);
10707 %}
10708
10709 ins_pipe(ialu_reg_reg_shift);
10710 %}
10711
10712 // This pattern is automatically generated from aarch64_ad.m4
10713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10714 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10715 iRegL src1, iRegL src2,
10716 immI src3) %{
10717 match(Set dst (AndL src1 (LShiftL src2 src3)));
10718
10719 ins_cost(1.9 * INSN_COST);
10720 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10721
10722 ins_encode %{
10723 __ andr(as_Register($dst$$reg),
10724 as_Register($src1$$reg),
10725 as_Register($src2$$reg),
10726 Assembler::LSL,
10727 $src3$$constant & 0x3f);
10728 %}
10729
10730 ins_pipe(ialu_reg_reg_shift);
10731 %}
10732
10733 // This pattern is automatically generated from aarch64_ad.m4
10734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10735 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10736 iRegIorL2I src1, iRegIorL2I src2,
10737 immI src3) %{
10738 match(Set dst (AndI src1 (RotateRight src2 src3)));
10739
10740 ins_cost(1.9 * INSN_COST);
10741 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10742
10743 ins_encode %{
10744 __ andw(as_Register($dst$$reg),
10745 as_Register($src1$$reg),
10746 as_Register($src2$$reg),
10747 Assembler::ROR,
10748 $src3$$constant & 0x1f);
10749 %}
10750
10751 ins_pipe(ialu_reg_reg_shift);
10752 %}
10753
10754 // This pattern is automatically generated from aarch64_ad.m4
10755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10756 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10757 iRegL src1, iRegL src2,
10758 immI src3) %{
10759 match(Set dst (AndL src1 (RotateRight src2 src3)));
10760
10761 ins_cost(1.9 * INSN_COST);
10762 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10763
10764 ins_encode %{
10765 __ andr(as_Register($dst$$reg),
10766 as_Register($src1$$reg),
10767 as_Register($src2$$reg),
10768 Assembler::ROR,
10769 $src3$$constant & 0x3f);
10770 %}
10771
10772 ins_pipe(ialu_reg_reg_shift);
10773 %}
10774
10775 // This pattern is automatically generated from aarch64_ad.m4
10776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10777 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10778 iRegIorL2I src1, iRegIorL2I src2,
10779 immI src3) %{
10780 match(Set dst (XorI src1 (URShiftI src2 src3)));
10781
10782 ins_cost(1.9 * INSN_COST);
10783 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10784
10785 ins_encode %{
10786 __ eorw(as_Register($dst$$reg),
10787 as_Register($src1$$reg),
10788 as_Register($src2$$reg),
10789 Assembler::LSR,
10790 $src3$$constant & 0x1f);
10791 %}
10792
10793 ins_pipe(ialu_reg_reg_shift);
10794 %}
10795
10796 // This pattern is automatically generated from aarch64_ad.m4
10797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10798 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10799 iRegL src1, iRegL src2,
10800 immI src3) %{
10801 match(Set dst (XorL src1 (URShiftL src2 src3)));
10802
10803 ins_cost(1.9 * INSN_COST);
10804 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10805
10806 ins_encode %{
10807 __ eor(as_Register($dst$$reg),
10808 as_Register($src1$$reg),
10809 as_Register($src2$$reg),
10810 Assembler::LSR,
10811 $src3$$constant & 0x3f);
10812 %}
10813
10814 ins_pipe(ialu_reg_reg_shift);
10815 %}
10816
10817 // This pattern is automatically generated from aarch64_ad.m4
10818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10819 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10820 iRegIorL2I src1, iRegIorL2I src2,
10821 immI src3) %{
10822 match(Set dst (XorI src1 (RShiftI src2 src3)));
10823
10824 ins_cost(1.9 * INSN_COST);
10825 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10826
10827 ins_encode %{
10828 __ eorw(as_Register($dst$$reg),
10829 as_Register($src1$$reg),
10830 as_Register($src2$$reg),
10831 Assembler::ASR,
10832 $src3$$constant & 0x1f);
10833 %}
10834
10835 ins_pipe(ialu_reg_reg_shift);
10836 %}
10837
10838 // This pattern is automatically generated from aarch64_ad.m4
10839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10840 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10841 iRegL src1, iRegL src2,
10842 immI src3) %{
10843 match(Set dst (XorL src1 (RShiftL src2 src3)));
10844
10845 ins_cost(1.9 * INSN_COST);
10846 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10847
10848 ins_encode %{
10849 __ eor(as_Register($dst$$reg),
10850 as_Register($src1$$reg),
10851 as_Register($src2$$reg),
10852 Assembler::ASR,
10853 $src3$$constant & 0x3f);
10854 %}
10855
10856 ins_pipe(ialu_reg_reg_shift);
10857 %}
10858
10859 // This pattern is automatically generated from aarch64_ad.m4
10860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10861 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10862 iRegIorL2I src1, iRegIorL2I src2,
10863 immI src3) %{
10864 match(Set dst (XorI src1 (LShiftI src2 src3)));
10865
10866 ins_cost(1.9 * INSN_COST);
10867 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10868
10869 ins_encode %{
10870 __ eorw(as_Register($dst$$reg),
10871 as_Register($src1$$reg),
10872 as_Register($src2$$reg),
10873 Assembler::LSL,
10874 $src3$$constant & 0x1f);
10875 %}
10876
10877 ins_pipe(ialu_reg_reg_shift);
10878 %}
10879
10880 // This pattern is automatically generated from aarch64_ad.m4
10881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10882 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10883 iRegL src1, iRegL src2,
10884 immI src3) %{
10885 match(Set dst (XorL src1 (LShiftL src2 src3)));
10886
10887 ins_cost(1.9 * INSN_COST);
10888 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10889
10890 ins_encode %{
10891 __ eor(as_Register($dst$$reg),
10892 as_Register($src1$$reg),
10893 as_Register($src2$$reg),
10894 Assembler::LSL,
10895 $src3$$constant & 0x3f);
10896 %}
10897
10898 ins_pipe(ialu_reg_reg_shift);
10899 %}
10900
10901 // This pattern is automatically generated from aarch64_ad.m4
10902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10903 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10904 iRegIorL2I src1, iRegIorL2I src2,
10905 immI src3) %{
10906 match(Set dst (XorI src1 (RotateRight src2 src3)));
10907
10908 ins_cost(1.9 * INSN_COST);
10909 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10910
10911 ins_encode %{
10912 __ eorw(as_Register($dst$$reg),
10913 as_Register($src1$$reg),
10914 as_Register($src2$$reg),
10915 Assembler::ROR,
10916 $src3$$constant & 0x1f);
10917 %}
10918
10919 ins_pipe(ialu_reg_reg_shift);
10920 %}
10921
10922 // This pattern is automatically generated from aarch64_ad.m4
10923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10924 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10925 iRegL src1, iRegL src2,
10926 immI src3) %{
10927 match(Set dst (XorL src1 (RotateRight src2 src3)));
10928
10929 ins_cost(1.9 * INSN_COST);
10930 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10931
10932 ins_encode %{
10933 __ eor(as_Register($dst$$reg),
10934 as_Register($src1$$reg),
10935 as_Register($src2$$reg),
10936 Assembler::ROR,
10937 $src3$$constant & 0x3f);
10938 %}
10939
10940 ins_pipe(ialu_reg_reg_shift);
10941 %}
10942
10943 // This pattern is automatically generated from aarch64_ad.m4
10944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10945 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10946 iRegIorL2I src1, iRegIorL2I src2,
10947 immI src3) %{
10948 match(Set dst (OrI src1 (URShiftI src2 src3)));
10949
10950 ins_cost(1.9 * INSN_COST);
10951 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10952
10953 ins_encode %{
10954 __ orrw(as_Register($dst$$reg),
10955 as_Register($src1$$reg),
10956 as_Register($src2$$reg),
10957 Assembler::LSR,
10958 $src3$$constant & 0x1f);
10959 %}
10960
10961 ins_pipe(ialu_reg_reg_shift);
10962 %}
10963
10964 // This pattern is automatically generated from aarch64_ad.m4
10965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10966 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10967 iRegL src1, iRegL src2,
10968 immI src3) %{
10969 match(Set dst (OrL src1 (URShiftL src2 src3)));
10970
10971 ins_cost(1.9 * INSN_COST);
10972 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10973
10974 ins_encode %{
10975 __ orr(as_Register($dst$$reg),
10976 as_Register($src1$$reg),
10977 as_Register($src2$$reg),
10978 Assembler::LSR,
10979 $src3$$constant & 0x3f);
10980 %}
10981
10982 ins_pipe(ialu_reg_reg_shift);
10983 %}
10984
10985 // This pattern is automatically generated from aarch64_ad.m4
10986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10987 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10988 iRegIorL2I src1, iRegIorL2I src2,
10989 immI src3) %{
10990 match(Set dst (OrI src1 (RShiftI src2 src3)));
10991
10992 ins_cost(1.9 * INSN_COST);
10993 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10994
10995 ins_encode %{
10996 __ orrw(as_Register($dst$$reg),
10997 as_Register($src1$$reg),
10998 as_Register($src2$$reg),
10999 Assembler::ASR,
11000 $src3$$constant & 0x1f);
11001 %}
11002
11003 ins_pipe(ialu_reg_reg_shift);
11004 %}
11005
11006 // This pattern is automatically generated from aarch64_ad.m4
11007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11008 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11009 iRegL src1, iRegL src2,
11010 immI src3) %{
11011 match(Set dst (OrL src1 (RShiftL src2 src3)));
11012
11013 ins_cost(1.9 * INSN_COST);
11014 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11015
11016 ins_encode %{
11017 __ orr(as_Register($dst$$reg),
11018 as_Register($src1$$reg),
11019 as_Register($src2$$reg),
11020 Assembler::ASR,
11021 $src3$$constant & 0x3f);
11022 %}
11023
11024 ins_pipe(ialu_reg_reg_shift);
11025 %}
11026
11027 // This pattern is automatically generated from aarch64_ad.m4
11028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11029 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11030 iRegIorL2I src1, iRegIorL2I src2,
11031 immI src3) %{
11032 match(Set dst (OrI src1 (LShiftI src2 src3)));
11033
11034 ins_cost(1.9 * INSN_COST);
11035 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11036
11037 ins_encode %{
11038 __ orrw(as_Register($dst$$reg),
11039 as_Register($src1$$reg),
11040 as_Register($src2$$reg),
11041 Assembler::LSL,
11042 $src3$$constant & 0x1f);
11043 %}
11044
11045 ins_pipe(ialu_reg_reg_shift);
11046 %}
11047
11048 // This pattern is automatically generated from aarch64_ad.m4
11049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11050 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11051 iRegL src1, iRegL src2,
11052 immI src3) %{
11053 match(Set dst (OrL src1 (LShiftL src2 src3)));
11054
11055 ins_cost(1.9 * INSN_COST);
11056 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11057
11058 ins_encode %{
11059 __ orr(as_Register($dst$$reg),
11060 as_Register($src1$$reg),
11061 as_Register($src2$$reg),
11062 Assembler::LSL,
11063 $src3$$constant & 0x3f);
11064 %}
11065
11066 ins_pipe(ialu_reg_reg_shift);
11067 %}
11068
11069 // This pattern is automatically generated from aarch64_ad.m4
11070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11071 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11072 iRegIorL2I src1, iRegIorL2I src2,
11073 immI src3) %{
11074 match(Set dst (OrI src1 (RotateRight src2 src3)));
11075
11076 ins_cost(1.9 * INSN_COST);
11077 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11078
11079 ins_encode %{
11080 __ orrw(as_Register($dst$$reg),
11081 as_Register($src1$$reg),
11082 as_Register($src2$$reg),
11083 Assembler::ROR,
11084 $src3$$constant & 0x1f);
11085 %}
11086
11087 ins_pipe(ialu_reg_reg_shift);
11088 %}
11089
11090 // This pattern is automatically generated from aarch64_ad.m4
11091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11092 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11093 iRegL src1, iRegL src2,
11094 immI src3) %{
11095 match(Set dst (OrL src1 (RotateRight src2 src3)));
11096
11097 ins_cost(1.9 * INSN_COST);
11098 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11099
11100 ins_encode %{
11101 __ orr(as_Register($dst$$reg),
11102 as_Register($src1$$reg),
11103 as_Register($src2$$reg),
11104 Assembler::ROR,
11105 $src3$$constant & 0x3f);
11106 %}
11107
11108 ins_pipe(ialu_reg_reg_shift);
11109 %}
11110
11111 // This pattern is automatically generated from aarch64_ad.m4
11112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11113 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11114 iRegIorL2I src1, iRegIorL2I src2,
11115 immI src3) %{
11116 match(Set dst (AddI src1 (URShiftI src2 src3)));
11117
11118 ins_cost(1.9 * INSN_COST);
11119 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11120
11121 ins_encode %{
11122 __ addw(as_Register($dst$$reg),
11123 as_Register($src1$$reg),
11124 as_Register($src2$$reg),
11125 Assembler::LSR,
11126 $src3$$constant & 0x1f);
11127 %}
11128
11129 ins_pipe(ialu_reg_reg_shift);
11130 %}
11131
11132 // This pattern is automatically generated from aarch64_ad.m4
11133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11134 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11135 iRegL src1, iRegL src2,
11136 immI src3) %{
11137 match(Set dst (AddL src1 (URShiftL src2 src3)));
11138
11139 ins_cost(1.9 * INSN_COST);
11140 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11141
11142 ins_encode %{
11143 __ add(as_Register($dst$$reg),
11144 as_Register($src1$$reg),
11145 as_Register($src2$$reg),
11146 Assembler::LSR,
11147 $src3$$constant & 0x3f);
11148 %}
11149
11150 ins_pipe(ialu_reg_reg_shift);
11151 %}
11152
11153 // This pattern is automatically generated from aarch64_ad.m4
11154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11155 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11156 iRegIorL2I src1, iRegIorL2I src2,
11157 immI src3) %{
11158 match(Set dst (AddI src1 (RShiftI src2 src3)));
11159
11160 ins_cost(1.9 * INSN_COST);
11161 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11162
11163 ins_encode %{
11164 __ addw(as_Register($dst$$reg),
11165 as_Register($src1$$reg),
11166 as_Register($src2$$reg),
11167 Assembler::ASR,
11168 $src3$$constant & 0x1f);
11169 %}
11170
11171 ins_pipe(ialu_reg_reg_shift);
11172 %}
11173
11174 // This pattern is automatically generated from aarch64_ad.m4
11175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11176 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11177 iRegL src1, iRegL src2,
11178 immI src3) %{
11179 match(Set dst (AddL src1 (RShiftL src2 src3)));
11180
11181 ins_cost(1.9 * INSN_COST);
11182 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11183
11184 ins_encode %{
11185 __ add(as_Register($dst$$reg),
11186 as_Register($src1$$reg),
11187 as_Register($src2$$reg),
11188 Assembler::ASR,
11189 $src3$$constant & 0x3f);
11190 %}
11191
11192 ins_pipe(ialu_reg_reg_shift);
11193 %}
11194
11195 // This pattern is automatically generated from aarch64_ad.m4
11196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11197 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11198 iRegIorL2I src1, iRegIorL2I src2,
11199 immI src3) %{
11200 match(Set dst (AddI src1 (LShiftI src2 src3)));
11201
11202 ins_cost(1.9 * INSN_COST);
11203 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11204
11205 ins_encode %{
11206 __ addw(as_Register($dst$$reg),
11207 as_Register($src1$$reg),
11208 as_Register($src2$$reg),
11209 Assembler::LSL,
11210 $src3$$constant & 0x1f);
11211 %}
11212
11213 ins_pipe(ialu_reg_reg_shift);
11214 %}
11215
11216 // This pattern is automatically generated from aarch64_ad.m4
11217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11218 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11219 iRegL src1, iRegL src2,
11220 immI src3) %{
11221 match(Set dst (AddL src1 (LShiftL src2 src3)));
11222
11223 ins_cost(1.9 * INSN_COST);
11224 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11225
11226 ins_encode %{
11227 __ add(as_Register($dst$$reg),
11228 as_Register($src1$$reg),
11229 as_Register($src2$$reg),
11230 Assembler::LSL,
11231 $src3$$constant & 0x3f);
11232 %}
11233
11234 ins_pipe(ialu_reg_reg_shift);
11235 %}
11236
11237 // This pattern is automatically generated from aarch64_ad.m4
11238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11239 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11240 iRegIorL2I src1, iRegIorL2I src2,
11241 immI src3) %{
11242 match(Set dst (SubI src1 (URShiftI src2 src3)));
11243
11244 ins_cost(1.9 * INSN_COST);
11245 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11246
11247 ins_encode %{
11248 __ subw(as_Register($dst$$reg),
11249 as_Register($src1$$reg),
11250 as_Register($src2$$reg),
11251 Assembler::LSR,
11252 $src3$$constant & 0x1f);
11253 %}
11254
11255 ins_pipe(ialu_reg_reg_shift);
11256 %}
11257
11258 // This pattern is automatically generated from aarch64_ad.m4
11259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11260 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11261 iRegL src1, iRegL src2,
11262 immI src3) %{
11263 match(Set dst (SubL src1 (URShiftL src2 src3)));
11264
11265 ins_cost(1.9 * INSN_COST);
11266 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11267
11268 ins_encode %{
11269 __ sub(as_Register($dst$$reg),
11270 as_Register($src1$$reg),
11271 as_Register($src2$$reg),
11272 Assembler::LSR,
11273 $src3$$constant & 0x3f);
11274 %}
11275
11276 ins_pipe(ialu_reg_reg_shift);
11277 %}
11278
11279 // This pattern is automatically generated from aarch64_ad.m4
11280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11281 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11282 iRegIorL2I src1, iRegIorL2I src2,
11283 immI src3) %{
11284 match(Set dst (SubI src1 (RShiftI src2 src3)));
11285
11286 ins_cost(1.9 * INSN_COST);
11287 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11288
11289 ins_encode %{
11290 __ subw(as_Register($dst$$reg),
11291 as_Register($src1$$reg),
11292 as_Register($src2$$reg),
11293 Assembler::ASR,
11294 $src3$$constant & 0x1f);
11295 %}
11296
11297 ins_pipe(ialu_reg_reg_shift);
11298 %}
11299
11300 // This pattern is automatically generated from aarch64_ad.m4
11301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11302 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11303 iRegL src1, iRegL src2,
11304 immI src3) %{
11305 match(Set dst (SubL src1 (RShiftL src2 src3)));
11306
11307 ins_cost(1.9 * INSN_COST);
11308 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11309
11310 ins_encode %{
11311 __ sub(as_Register($dst$$reg),
11312 as_Register($src1$$reg),
11313 as_Register($src2$$reg),
11314 Assembler::ASR,
11315 $src3$$constant & 0x3f);
11316 %}
11317
11318 ins_pipe(ialu_reg_reg_shift);
11319 %}
11320
11321 // This pattern is automatically generated from aarch64_ad.m4
11322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11323 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11324 iRegIorL2I src1, iRegIorL2I src2,
11325 immI src3) %{
11326 match(Set dst (SubI src1 (LShiftI src2 src3)));
11327
11328 ins_cost(1.9 * INSN_COST);
11329 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11330
11331 ins_encode %{
11332 __ subw(as_Register($dst$$reg),
11333 as_Register($src1$$reg),
11334 as_Register($src2$$reg),
11335 Assembler::LSL,
11336 $src3$$constant & 0x1f);
11337 %}
11338
11339 ins_pipe(ialu_reg_reg_shift);
11340 %}
11341
11342 // This pattern is automatically generated from aarch64_ad.m4
11343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11344 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11345 iRegL src1, iRegL src2,
11346 immI src3) %{
11347 match(Set dst (SubL src1 (LShiftL src2 src3)));
11348
11349 ins_cost(1.9 * INSN_COST);
11350 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11351
11352 ins_encode %{
11353 __ sub(as_Register($dst$$reg),
11354 as_Register($src1$$reg),
11355 as_Register($src2$$reg),
11356 Assembler::LSL,
11357 $src3$$constant & 0x3f);
11358 %}
11359
11360 ins_pipe(ialu_reg_reg_shift);
11361 %}
11362
11363 // This pattern is automatically generated from aarch64_ad.m4
11364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11365
11366 // Shift Left followed by Shift Right.
11367 // This idiom is used by the compiler for the i2b bytecode etc.
11368 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11369 %{
11370 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11371 ins_cost(INSN_COST * 2);
11372 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11373 ins_encode %{
11374 int lshift = $lshift_count$$constant & 63;
11375 int rshift = $rshift_count$$constant & 63;
11376 int s = 63 - lshift;
11377 int r = (rshift - lshift) & 63;
11378 __ sbfm(as_Register($dst$$reg),
11379 as_Register($src$$reg),
11380 r, s);
11381 %}
11382
11383 ins_pipe(ialu_reg_shift);
11384 %}
11385
11386 // This pattern is automatically generated from aarch64_ad.m4
11387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11388
11389 // Shift Left followed by Shift Right.
11390 // This idiom is used by the compiler for the i2b bytecode etc.
11391 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11392 %{
11393 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11394 ins_cost(INSN_COST * 2);
11395 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11396 ins_encode %{
11397 int lshift = $lshift_count$$constant & 31;
11398 int rshift = $rshift_count$$constant & 31;
11399 int s = 31 - lshift;
11400 int r = (rshift - lshift) & 31;
11401 __ sbfmw(as_Register($dst$$reg),
11402 as_Register($src$$reg),
11403 r, s);
11404 %}
11405
11406 ins_pipe(ialu_reg_shift);
11407 %}
11408
11409 // This pattern is automatically generated from aarch64_ad.m4
11410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11411
11412 // Shift Left followed by Shift Right.
11413 // This idiom is used by the compiler for the i2b bytecode etc.
11414 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11415 %{
11416 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11417 ins_cost(INSN_COST * 2);
11418 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11419 ins_encode %{
11420 int lshift = $lshift_count$$constant & 63;
11421 int rshift = $rshift_count$$constant & 63;
11422 int s = 63 - lshift;
11423 int r = (rshift - lshift) & 63;
11424 __ ubfm(as_Register($dst$$reg),
11425 as_Register($src$$reg),
11426 r, s);
11427 %}
11428
11429 ins_pipe(ialu_reg_shift);
11430 %}
11431
11432 // This pattern is automatically generated from aarch64_ad.m4
11433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11434
11435 // Shift Left followed by Shift Right.
11436 // This idiom is used by the compiler for the i2b bytecode etc.
11437 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11438 %{
11439 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11440 ins_cost(INSN_COST * 2);
11441 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11442 ins_encode %{
11443 int lshift = $lshift_count$$constant & 31;
11444 int rshift = $rshift_count$$constant & 31;
11445 int s = 31 - lshift;
11446 int r = (rshift - lshift) & 31;
11447 __ ubfmw(as_Register($dst$$reg),
11448 as_Register($src$$reg),
11449 r, s);
11450 %}
11451
11452 ins_pipe(ialu_reg_shift);
11453 %}
11454
11455 // Bitfield extract with shift & mask
11456
11457 // This pattern is automatically generated from aarch64_ad.m4
11458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11459 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11460 %{
11461 match(Set dst (AndI (URShiftI src rshift) mask));
11462 // Make sure we are not going to exceed what ubfxw can do.
11463 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11464
11465 ins_cost(INSN_COST);
11466 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11467 ins_encode %{
11468 int rshift = $rshift$$constant & 31;
11469 intptr_t mask = $mask$$constant;
11470 int width = exact_log2(mask+1);
11471 __ ubfxw(as_Register($dst$$reg),
11472 as_Register($src$$reg), rshift, width);
11473 %}
11474 ins_pipe(ialu_reg_shift);
11475 %}
11476
11477 // This pattern is automatically generated from aarch64_ad.m4
11478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11479 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11480 %{
11481 match(Set dst (AndL (URShiftL src rshift) mask));
11482 // Make sure we are not going to exceed what ubfx can do.
11483 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11484
11485 ins_cost(INSN_COST);
11486 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11487 ins_encode %{
11488 int rshift = $rshift$$constant & 63;
11489 intptr_t mask = $mask$$constant;
11490 int width = exact_log2_long(mask+1);
11491 __ ubfx(as_Register($dst$$reg),
11492 as_Register($src$$reg), rshift, width);
11493 %}
11494 ins_pipe(ialu_reg_shift);
11495 %}
11496
11497
11498 // This pattern is automatically generated from aarch64_ad.m4
11499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11500
11501 // We can use ubfx when extending an And with a mask when we know mask
11502 // is positive. We know that because immI_bitmask guarantees it.
11503 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11504 %{
11505 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11506 // Make sure we are not going to exceed what ubfxw can do.
11507 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11508
11509 ins_cost(INSN_COST * 2);
11510 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11511 ins_encode %{
11512 int rshift = $rshift$$constant & 31;
11513 intptr_t mask = $mask$$constant;
11514 int width = exact_log2(mask+1);
11515 __ ubfx(as_Register($dst$$reg),
11516 as_Register($src$$reg), rshift, width);
11517 %}
11518 ins_pipe(ialu_reg_shift);
11519 %}
11520
11521
11522 // This pattern is automatically generated from aarch64_ad.m4
11523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11524
11525 // We can use ubfiz when masking by a positive number and then left shifting the result.
11526 // We know that the mask is positive because immI_bitmask guarantees it.
11527 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11528 %{
11529 match(Set dst (LShiftI (AndI src mask) lshift));
11530 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11531
11532 ins_cost(INSN_COST);
11533 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11534 ins_encode %{
11535 int lshift = $lshift$$constant & 31;
11536 intptr_t mask = $mask$$constant;
11537 int width = exact_log2(mask+1);
11538 __ ubfizw(as_Register($dst$$reg),
11539 as_Register($src$$reg), lshift, width);
11540 %}
11541 ins_pipe(ialu_reg_shift);
11542 %}
11543
11544 // This pattern is automatically generated from aarch64_ad.m4
11545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11546
11547 // We can use ubfiz when masking by a positive number and then left shifting the result.
11548 // We know that the mask is positive because immL_bitmask guarantees it.
11549 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11550 %{
11551 match(Set dst (LShiftL (AndL src mask) lshift));
11552 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11553
11554 ins_cost(INSN_COST);
11555 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11556 ins_encode %{
11557 int lshift = $lshift$$constant & 63;
11558 intptr_t mask = $mask$$constant;
11559 int width = exact_log2_long(mask+1);
11560 __ ubfiz(as_Register($dst$$reg),
11561 as_Register($src$$reg), lshift, width);
11562 %}
11563 ins_pipe(ialu_reg_shift);
11564 %}
11565
11566 // This pattern is automatically generated from aarch64_ad.m4
11567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11568
11569 // We can use ubfiz when masking by a positive number and then left shifting the result.
11570 // We know that the mask is positive because immI_bitmask guarantees it.
11571 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11572 %{
11573 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11574 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11575
11576 ins_cost(INSN_COST);
11577 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11578 ins_encode %{
11579 int lshift = $lshift$$constant & 31;
11580 intptr_t mask = $mask$$constant;
11581 int width = exact_log2(mask+1);
11582 __ ubfizw(as_Register($dst$$reg),
11583 as_Register($src$$reg), lshift, width);
11584 %}
11585 ins_pipe(ialu_reg_shift);
11586 %}
11587
11588 // This pattern is automatically generated from aarch64_ad.m4
11589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11590
11591 // We can use ubfiz when masking by a positive number and then left shifting the result.
11592 // We know that the mask is positive because immL_bitmask guarantees it.
11593 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11594 %{
11595 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11596 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11597
11598 ins_cost(INSN_COST);
11599 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11600 ins_encode %{
11601 int lshift = $lshift$$constant & 63;
11602 intptr_t mask = $mask$$constant;
11603 int width = exact_log2_long(mask+1);
11604 __ ubfiz(as_Register($dst$$reg),
11605 as_Register($src$$reg), lshift, width);
11606 %}
11607 ins_pipe(ialu_reg_shift);
11608 %}
11609
11610
11611 // This pattern is automatically generated from aarch64_ad.m4
11612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11613
11614 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11615 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11616 %{
11617 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11618 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11619
11620 ins_cost(INSN_COST);
11621 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11622 ins_encode %{
11623 int lshift = $lshift$$constant & 63;
11624 intptr_t mask = $mask$$constant;
11625 int width = exact_log2(mask+1);
11626 __ ubfiz(as_Register($dst$$reg),
11627 as_Register($src$$reg), lshift, width);
11628 %}
11629 ins_pipe(ialu_reg_shift);
11630 %}
11631
11632 // This pattern is automatically generated from aarch64_ad.m4
11633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11634
11635 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11636 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11637 %{
11638 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11639 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11640
11641 ins_cost(INSN_COST);
11642 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11643 ins_encode %{
11644 int lshift = $lshift$$constant & 31;
11645 intptr_t mask = $mask$$constant;
11646 int width = exact_log2(mask+1);
11647 __ ubfiz(as_Register($dst$$reg),
11648 as_Register($src$$reg), lshift, width);
11649 %}
11650 ins_pipe(ialu_reg_shift);
11651 %}
11652
11653 // This pattern is automatically generated from aarch64_ad.m4
11654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11655
11656 // Can skip int2long conversions after AND with small bitmask
11657 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11658 %{
11659 match(Set dst (ConvI2L (AndI src msk)));
11660 ins_cost(INSN_COST);
11661 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11662 ins_encode %{
11663 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11664 %}
11665 ins_pipe(ialu_reg_shift);
11666 %}
11667
11668
11669 // Rotations
11670
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11674 %{
11675 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11676 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11677
11678 ins_cost(INSN_COST);
11679 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11680
11681 ins_encode %{
11682 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11683 $rshift$$constant & 63);
11684 %}
11685 ins_pipe(ialu_reg_reg_extr);
11686 %}
11687
11688
11689 // This pattern is automatically generated from aarch64_ad.m4
11690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11691 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11692 %{
11693 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11694 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11695
11696 ins_cost(INSN_COST);
11697 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11698
11699 ins_encode %{
11700 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11701 $rshift$$constant & 31);
11702 %}
11703 ins_pipe(ialu_reg_reg_extr);
11704 %}
11705
11706
11707 // This pattern is automatically generated from aarch64_ad.m4
11708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11709 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11710 %{
11711 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11712 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11713
11714 ins_cost(INSN_COST);
11715 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11716
11717 ins_encode %{
11718 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11719 $rshift$$constant & 63);
11720 %}
11721 ins_pipe(ialu_reg_reg_extr);
11722 %}
11723
11724
11725 // This pattern is automatically generated from aarch64_ad.m4
11726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11727 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11728 %{
11729 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11730 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11731
11732 ins_cost(INSN_COST);
11733 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11734
11735 ins_encode %{
11736 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11737 $rshift$$constant & 31);
11738 %}
11739 ins_pipe(ialu_reg_reg_extr);
11740 %}
11741
11742 // This pattern is automatically generated from aarch64_ad.m4
11743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11744 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11745 %{
11746 match(Set dst (RotateRight src shift));
11747
11748 ins_cost(INSN_COST);
11749 format %{ "ror $dst, $src, $shift" %}
11750
11751 ins_encode %{
11752 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11753 $shift$$constant & 0x1f);
11754 %}
11755 ins_pipe(ialu_reg_reg_vshift);
11756 %}
11757
11758 // This pattern is automatically generated from aarch64_ad.m4
11759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11760 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11761 %{
11762 match(Set dst (RotateRight src shift));
11763
11764 ins_cost(INSN_COST);
11765 format %{ "ror $dst, $src, $shift" %}
11766
11767 ins_encode %{
11768 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11769 $shift$$constant & 0x3f);
11770 %}
11771 ins_pipe(ialu_reg_reg_vshift);
11772 %}
11773
11774 // This pattern is automatically generated from aarch64_ad.m4
11775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11776 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11777 %{
11778 match(Set dst (RotateRight src shift));
11779
11780 ins_cost(INSN_COST);
11781 format %{ "ror $dst, $src, $shift" %}
11782
11783 ins_encode %{
11784 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11785 %}
11786 ins_pipe(ialu_reg_reg_vshift);
11787 %}
11788
11789 // This pattern is automatically generated from aarch64_ad.m4
11790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11791 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11792 %{
11793 match(Set dst (RotateRight src shift));
11794
11795 ins_cost(INSN_COST);
11796 format %{ "ror $dst, $src, $shift" %}
11797
11798 ins_encode %{
11799 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11800 %}
11801 ins_pipe(ialu_reg_reg_vshift);
11802 %}
11803
11804 // This pattern is automatically generated from aarch64_ad.m4
11805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11806 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11807 %{
11808 match(Set dst (RotateLeft src shift));
11809
11810 ins_cost(INSN_COST);
11811 format %{ "rol $dst, $src, $shift" %}
11812
11813 ins_encode %{
11814 __ subw(rscratch1, zr, as_Register($shift$$reg));
11815 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11816 %}
11817 ins_pipe(ialu_reg_reg_vshift);
11818 %}
11819
11820 // This pattern is automatically generated from aarch64_ad.m4
11821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11822 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11823 %{
11824 match(Set dst (RotateLeft src shift));
11825
11826 ins_cost(INSN_COST);
11827 format %{ "rol $dst, $src, $shift" %}
11828
11829 ins_encode %{
11830 __ subw(rscratch1, zr, as_Register($shift$$reg));
11831 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11832 %}
11833 ins_pipe(ialu_reg_reg_vshift);
11834 %}
11835
11836
11837 // Add/subtract (extended)
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11842 %{
11843 match(Set dst (AddL src1 (ConvI2L src2)));
11844 ins_cost(INSN_COST);
11845 format %{ "add $dst, $src1, $src2, sxtw" %}
11846
11847 ins_encode %{
11848 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11849 as_Register($src2$$reg), ext::sxtw);
11850 %}
11851 ins_pipe(ialu_reg_reg);
11852 %}
11853
11854 // This pattern is automatically generated from aarch64_ad.m4
11855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11856 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11857 %{
11858 match(Set dst (SubL src1 (ConvI2L src2)));
11859 ins_cost(INSN_COST);
11860 format %{ "sub $dst, $src1, $src2, sxtw" %}
11861
11862 ins_encode %{
11863 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11864 as_Register($src2$$reg), ext::sxtw);
11865 %}
11866 ins_pipe(ialu_reg_reg);
11867 %}
11868
11869 // This pattern is automatically generated from aarch64_ad.m4
11870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11871 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11872 %{
11873 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11874 ins_cost(INSN_COST);
11875 format %{ "add $dst, $src1, $src2, sxth" %}
11876
11877 ins_encode %{
11878 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11879 as_Register($src2$$reg), ext::sxth);
11880 %}
11881 ins_pipe(ialu_reg_reg);
11882 %}
11883
11884 // This pattern is automatically generated from aarch64_ad.m4
11885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11886 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11887 %{
11888 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11889 ins_cost(INSN_COST);
11890 format %{ "add $dst, $src1, $src2, sxtb" %}
11891
11892 ins_encode %{
11893 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11894 as_Register($src2$$reg), ext::sxtb);
11895 %}
11896 ins_pipe(ialu_reg_reg);
11897 %}
11898
11899 // This pattern is automatically generated from aarch64_ad.m4
11900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11901 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11902 %{
11903 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11904 ins_cost(INSN_COST);
11905 format %{ "add $dst, $src1, $src2, uxtb" %}
11906
11907 ins_encode %{
11908 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11909 as_Register($src2$$reg), ext::uxtb);
11910 %}
11911 ins_pipe(ialu_reg_reg);
11912 %}
11913
11914 // This pattern is automatically generated from aarch64_ad.m4
11915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11916 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11917 %{
11918 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11919 ins_cost(INSN_COST);
11920 format %{ "add $dst, $src1, $src2, sxth" %}
11921
11922 ins_encode %{
11923 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11924 as_Register($src2$$reg), ext::sxth);
11925 %}
11926 ins_pipe(ialu_reg_reg);
11927 %}
11928
11929 // This pattern is automatically generated from aarch64_ad.m4
11930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11931 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11932 %{
11933 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11934 ins_cost(INSN_COST);
11935 format %{ "add $dst, $src1, $src2, sxtw" %}
11936
11937 ins_encode %{
11938 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11939 as_Register($src2$$reg), ext::sxtw);
11940 %}
11941 ins_pipe(ialu_reg_reg);
11942 %}
11943
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11947 %{
11948 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11949 ins_cost(INSN_COST);
11950 format %{ "add $dst, $src1, $src2, sxtb" %}
11951
11952 ins_encode %{
11953 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11954 as_Register($src2$$reg), ext::sxtb);
11955 %}
11956 ins_pipe(ialu_reg_reg);
11957 %}
11958
11959 // This pattern is automatically generated from aarch64_ad.m4
11960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11961 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11962 %{
11963 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11964 ins_cost(INSN_COST);
11965 format %{ "add $dst, $src1, $src2, uxtb" %}
11966
11967 ins_encode %{
11968 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11969 as_Register($src2$$reg), ext::uxtb);
11970 %}
11971 ins_pipe(ialu_reg_reg);
11972 %}
11973
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11977 %{
11978 match(Set dst (AddI src1 (AndI src2 mask)));
11979 ins_cost(INSN_COST);
11980 format %{ "addw $dst, $src1, $src2, uxtb" %}
11981
11982 ins_encode %{
11983 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11984 as_Register($src2$$reg), ext::uxtb);
11985 %}
11986 ins_pipe(ialu_reg_reg);
11987 %}
11988
11989 // This pattern is automatically generated from aarch64_ad.m4
11990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11991 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11992 %{
11993 match(Set dst (AddI src1 (AndI src2 mask)));
11994 ins_cost(INSN_COST);
11995 format %{ "addw $dst, $src1, $src2, uxth" %}
11996
11997 ins_encode %{
11998 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11999 as_Register($src2$$reg), ext::uxth);
12000 %}
12001 ins_pipe(ialu_reg_reg);
12002 %}
12003
12004 // This pattern is automatically generated from aarch64_ad.m4
12005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12006 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12007 %{
12008 match(Set dst (AddL src1 (AndL src2 mask)));
12009 ins_cost(INSN_COST);
12010 format %{ "add $dst, $src1, $src2, uxtb" %}
12011
12012 ins_encode %{
12013 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12014 as_Register($src2$$reg), ext::uxtb);
12015 %}
12016 ins_pipe(ialu_reg_reg);
12017 %}
12018
12019 // This pattern is automatically generated from aarch64_ad.m4
12020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12021 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12022 %{
12023 match(Set dst (AddL src1 (AndL src2 mask)));
12024 ins_cost(INSN_COST);
12025 format %{ "add $dst, $src1, $src2, uxth" %}
12026
12027 ins_encode %{
12028 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12029 as_Register($src2$$reg), ext::uxth);
12030 %}
12031 ins_pipe(ialu_reg_reg);
12032 %}
12033
12034 // This pattern is automatically generated from aarch64_ad.m4
12035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12036 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12037 %{
12038 match(Set dst (AddL src1 (AndL src2 mask)));
12039 ins_cost(INSN_COST);
12040 format %{ "add $dst, $src1, $src2, uxtw" %}
12041
12042 ins_encode %{
12043 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12044 as_Register($src2$$reg), ext::uxtw);
12045 %}
12046 ins_pipe(ialu_reg_reg);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12052 %{
12053 match(Set dst (SubI src1 (AndI src2 mask)));
12054 ins_cost(INSN_COST);
12055 format %{ "subw $dst, $src1, $src2, uxtb" %}
12056
12057 ins_encode %{
12058 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12059 as_Register($src2$$reg), ext::uxtb);
12060 %}
12061 ins_pipe(ialu_reg_reg);
12062 %}
12063
12064 // This pattern is automatically generated from aarch64_ad.m4
12065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12066 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12067 %{
12068 match(Set dst (SubI src1 (AndI src2 mask)));
12069 ins_cost(INSN_COST);
12070 format %{ "subw $dst, $src1, $src2, uxth" %}
12071
12072 ins_encode %{
12073 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12074 as_Register($src2$$reg), ext::uxth);
12075 %}
12076 ins_pipe(ialu_reg_reg);
12077 %}
12078
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12082 %{
12083 match(Set dst (SubL src1 (AndL src2 mask)));
12084 ins_cost(INSN_COST);
12085 format %{ "sub $dst, $src1, $src2, uxtb" %}
12086
12087 ins_encode %{
12088 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12089 as_Register($src2$$reg), ext::uxtb);
12090 %}
12091 ins_pipe(ialu_reg_reg);
12092 %}
12093
12094 // This pattern is automatically generated from aarch64_ad.m4
12095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12096 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12097 %{
12098 match(Set dst (SubL src1 (AndL src2 mask)));
12099 ins_cost(INSN_COST);
12100 format %{ "sub $dst, $src1, $src2, uxth" %}
12101
12102 ins_encode %{
12103 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12104 as_Register($src2$$reg), ext::uxth);
12105 %}
12106 ins_pipe(ialu_reg_reg);
12107 %}
12108
12109 // This pattern is automatically generated from aarch64_ad.m4
12110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12111 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12112 %{
12113 match(Set dst (SubL src1 (AndL src2 mask)));
12114 ins_cost(INSN_COST);
12115 format %{ "sub $dst, $src1, $src2, uxtw" %}
12116
12117 ins_encode %{
12118 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12119 as_Register($src2$$reg), ext::uxtw);
12120 %}
12121 ins_pipe(ialu_reg_reg);
12122 %}
12123
12124
12125 // This pattern is automatically generated from aarch64_ad.m4
12126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12127 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12128 %{
12129 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12130 ins_cost(1.9 * INSN_COST);
12131 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12132
12133 ins_encode %{
12134 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12135 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12136 %}
12137 ins_pipe(ialu_reg_reg_shift);
12138 %}
12139
12140 // This pattern is automatically generated from aarch64_ad.m4
12141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12142 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12143 %{
12144 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12145 ins_cost(1.9 * INSN_COST);
12146 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12147
12148 ins_encode %{
12149 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12150 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12151 %}
12152 ins_pipe(ialu_reg_reg_shift);
12153 %}
12154
12155 // This pattern is automatically generated from aarch64_ad.m4
12156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12157 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12158 %{
12159 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12160 ins_cost(1.9 * INSN_COST);
12161 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12162
12163 ins_encode %{
12164 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12165 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12166 %}
12167 ins_pipe(ialu_reg_reg_shift);
12168 %}
12169
12170 // This pattern is automatically generated from aarch64_ad.m4
12171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12172 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12173 %{
12174 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12175 ins_cost(1.9 * INSN_COST);
12176 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12177
12178 ins_encode %{
12179 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12180 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12181 %}
12182 ins_pipe(ialu_reg_reg_shift);
12183 %}
12184
12185 // This pattern is automatically generated from aarch64_ad.m4
12186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12187 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12188 %{
12189 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12190 ins_cost(1.9 * INSN_COST);
12191 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12192
12193 ins_encode %{
12194 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12195 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12196 %}
12197 ins_pipe(ialu_reg_reg_shift);
12198 %}
12199
12200 // This pattern is automatically generated from aarch64_ad.m4
12201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12202 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12203 %{
12204 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12205 ins_cost(1.9 * INSN_COST);
12206 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12207
12208 ins_encode %{
12209 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12210 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12211 %}
12212 ins_pipe(ialu_reg_reg_shift);
12213 %}
12214
12215 // This pattern is automatically generated from aarch64_ad.m4
12216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12217 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12218 %{
12219 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12220 ins_cost(1.9 * INSN_COST);
12221 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12222
12223 ins_encode %{
12224 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12225 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12226 %}
12227 ins_pipe(ialu_reg_reg_shift);
12228 %}
12229
12230 // This pattern is automatically generated from aarch64_ad.m4
12231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12232 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12233 %{
12234 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12235 ins_cost(1.9 * INSN_COST);
12236 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12237
12238 ins_encode %{
12239 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12240 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12241 %}
12242 ins_pipe(ialu_reg_reg_shift);
12243 %}
12244
12245 // This pattern is automatically generated from aarch64_ad.m4
12246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12247 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12248 %{
12249 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12250 ins_cost(1.9 * INSN_COST);
12251 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12252
12253 ins_encode %{
12254 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12255 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12256 %}
12257 ins_pipe(ialu_reg_reg_shift);
12258 %}
12259
12260 // This pattern is automatically generated from aarch64_ad.m4
12261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12262 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12263 %{
12264 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12265 ins_cost(1.9 * INSN_COST);
12266 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12267
12268 ins_encode %{
12269 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12270 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12271 %}
12272 ins_pipe(ialu_reg_reg_shift);
12273 %}
12274
12275 // This pattern is automatically generated from aarch64_ad.m4
12276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12277 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12278 %{
12279 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12280 ins_cost(1.9 * INSN_COST);
12281 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12282
12283 ins_encode %{
12284 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12285 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12286 %}
12287 ins_pipe(ialu_reg_reg_shift);
12288 %}
12289
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12293 %{
12294 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12295 ins_cost(1.9 * INSN_COST);
12296 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12297
12298 ins_encode %{
12299 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12300 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12301 %}
12302 ins_pipe(ialu_reg_reg_shift);
12303 %}
12304
12305 // This pattern is automatically generated from aarch64_ad.m4
12306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12307 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12308 %{
12309 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12310 ins_cost(1.9 * INSN_COST);
12311 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12312
12313 ins_encode %{
12314 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12315 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12316 %}
12317 ins_pipe(ialu_reg_reg_shift);
12318 %}
12319
12320 // This pattern is automatically generated from aarch64_ad.m4
12321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12322 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12323 %{
12324 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12325 ins_cost(1.9 * INSN_COST);
12326 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12327
12328 ins_encode %{
12329 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12330 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12331 %}
12332 ins_pipe(ialu_reg_reg_shift);
12333 %}
12334
12335 // This pattern is automatically generated from aarch64_ad.m4
12336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12337 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12338 %{
12339 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12340 ins_cost(1.9 * INSN_COST);
12341 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12342
12343 ins_encode %{
12344 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12345 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12346 %}
12347 ins_pipe(ialu_reg_reg_shift);
12348 %}
12349
12350 // This pattern is automatically generated from aarch64_ad.m4
12351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12352 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12353 %{
12354 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12355 ins_cost(1.9 * INSN_COST);
12356 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12357
12358 ins_encode %{
12359 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12360 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12361 %}
12362 ins_pipe(ialu_reg_reg_shift);
12363 %}
12364
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12368 %{
12369 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12370 ins_cost(1.9 * INSN_COST);
12371 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12372
12373 ins_encode %{
12374 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12375 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12376 %}
12377 ins_pipe(ialu_reg_reg_shift);
12378 %}
12379
12380 // This pattern is automatically generated from aarch64_ad.m4
12381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12382 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12383 %{
12384 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12385 ins_cost(1.9 * INSN_COST);
12386 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12387
12388 ins_encode %{
12389 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12390 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12391 %}
12392 ins_pipe(ialu_reg_reg_shift);
12393 %}
12394
12395 // This pattern is automatically generated from aarch64_ad.m4
12396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12397 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12398 %{
12399 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12400 ins_cost(1.9 * INSN_COST);
12401 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12402
12403 ins_encode %{
12404 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12405 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12406 %}
12407 ins_pipe(ialu_reg_reg_shift);
12408 %}
12409
12410 // This pattern is automatically generated from aarch64_ad.m4
12411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12412 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12413 %{
12414 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12415 ins_cost(1.9 * INSN_COST);
12416 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12417
12418 ins_encode %{
12419 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12420 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12421 %}
12422 ins_pipe(ialu_reg_reg_shift);
12423 %}
12424
12425 // This pattern is automatically generated from aarch64_ad.m4
12426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12427 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12428 %{
12429 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12430 ins_cost(1.9 * INSN_COST);
12431 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12432
12433 ins_encode %{
12434 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12435 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12436 %}
12437 ins_pipe(ialu_reg_reg_shift);
12438 %}
12439
12440 // This pattern is automatically generated from aarch64_ad.m4
12441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12442 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12443 %{
12444 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12445 ins_cost(1.9 * INSN_COST);
12446 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12447
12448 ins_encode %{
12449 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12450 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12451 %}
12452 ins_pipe(ialu_reg_reg_shift);
12453 %}
12454
12455 // This pattern is automatically generated from aarch64_ad.m4
12456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12457 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12458 %{
12459 effect(DEF dst, USE src1, USE src2, USE cr);
12460 ins_cost(INSN_COST * 2);
12461 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12462
12463 ins_encode %{
12464 __ cselw($dst$$Register,
12465 $src1$$Register,
12466 $src2$$Register,
12467 Assembler::LT);
12468 %}
12469 ins_pipe(icond_reg_reg);
12470 %}
12471
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12475 %{
12476 effect(DEF dst, USE src1, USE src2, USE cr);
12477 ins_cost(INSN_COST * 2);
12478 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12479
12480 ins_encode %{
12481 __ cselw($dst$$Register,
12482 $src1$$Register,
12483 $src2$$Register,
12484 Assembler::GT);
12485 %}
12486 ins_pipe(icond_reg_reg);
12487 %}
12488
12489 // This pattern is automatically generated from aarch64_ad.m4
12490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12491 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12492 %{
12493 effect(DEF dst, USE src1, USE cr);
12494 ins_cost(INSN_COST * 2);
12495 format %{ "cselw $dst, $src1, zr lt\t" %}
12496
12497 ins_encode %{
12498 __ cselw($dst$$Register,
12499 $src1$$Register,
12500 zr,
12501 Assembler::LT);
12502 %}
12503 ins_pipe(icond_reg);
12504 %}
12505
12506 // This pattern is automatically generated from aarch64_ad.m4
12507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12508 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12509 %{
12510 effect(DEF dst, USE src1, USE cr);
12511 ins_cost(INSN_COST * 2);
12512 format %{ "cselw $dst, $src1, zr gt\t" %}
12513
12514 ins_encode %{
12515 __ cselw($dst$$Register,
12516 $src1$$Register,
12517 zr,
12518 Assembler::GT);
12519 %}
12520 ins_pipe(icond_reg);
12521 %}
12522
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12526 %{
12527 effect(DEF dst, USE src1, USE cr);
12528 ins_cost(INSN_COST * 2);
12529 format %{ "csincw $dst, $src1, zr le\t" %}
12530
12531 ins_encode %{
12532 __ csincw($dst$$Register,
12533 $src1$$Register,
12534 zr,
12535 Assembler::LE);
12536 %}
12537 ins_pipe(icond_reg);
12538 %}
12539
12540 // This pattern is automatically generated from aarch64_ad.m4
12541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12542 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12543 %{
12544 effect(DEF dst, USE src1, USE cr);
12545 ins_cost(INSN_COST * 2);
12546 format %{ "csincw $dst, $src1, zr gt\t" %}
12547
12548 ins_encode %{
12549 __ csincw($dst$$Register,
12550 $src1$$Register,
12551 zr,
12552 Assembler::GT);
12553 %}
12554 ins_pipe(icond_reg);
12555 %}
12556
12557 // This pattern is automatically generated from aarch64_ad.m4
12558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12559 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12560 %{
12561 effect(DEF dst, USE src1, USE cr);
12562 ins_cost(INSN_COST * 2);
12563 format %{ "csinvw $dst, $src1, zr lt\t" %}
12564
12565 ins_encode %{
12566 __ csinvw($dst$$Register,
12567 $src1$$Register,
12568 zr,
12569 Assembler::LT);
12570 %}
12571 ins_pipe(icond_reg);
12572 %}
12573
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12577 %{
12578 effect(DEF dst, USE src1, USE cr);
12579 ins_cost(INSN_COST * 2);
12580 format %{ "csinvw $dst, $src1, zr ge\t" %}
12581
12582 ins_encode %{
12583 __ csinvw($dst$$Register,
12584 $src1$$Register,
12585 zr,
12586 Assembler::GE);
12587 %}
12588 ins_pipe(icond_reg);
12589 %}
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12594 %{
12595 match(Set dst (MinI src imm));
12596 ins_cost(INSN_COST * 3);
12597 expand %{
12598 rFlagsReg cr;
12599 compI_reg_imm0(cr, src);
12600 cmovI_reg_imm0_lt(dst, src, cr);
12601 %}
12602 %}
12603
12604 // This pattern is automatically generated from aarch64_ad.m4
12605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12606 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12607 %{
12608 match(Set dst (MinI imm src));
12609 ins_cost(INSN_COST * 3);
12610 expand %{
12611 rFlagsReg cr;
12612 compI_reg_imm0(cr, src);
12613 cmovI_reg_imm0_lt(dst, src, cr);
12614 %}
12615 %}
12616
12617 // This pattern is automatically generated from aarch64_ad.m4
12618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12619 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12620 %{
12621 match(Set dst (MinI src imm));
12622 ins_cost(INSN_COST * 3);
12623 expand %{
12624 rFlagsReg cr;
12625 compI_reg_imm0(cr, src);
12626 cmovI_reg_imm1_le(dst, src, cr);
12627 %}
12628 %}
12629
12630 // This pattern is automatically generated from aarch64_ad.m4
12631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12632 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12633 %{
12634 match(Set dst (MinI imm src));
12635 ins_cost(INSN_COST * 3);
12636 expand %{
12637 rFlagsReg cr;
12638 compI_reg_imm0(cr, src);
12639 cmovI_reg_imm1_le(dst, src, cr);
12640 %}
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12646 %{
12647 match(Set dst (MinI src imm));
12648 ins_cost(INSN_COST * 3);
12649 expand %{
12650 rFlagsReg cr;
12651 compI_reg_imm0(cr, src);
12652 cmovI_reg_immM1_lt(dst, src, cr);
12653 %}
12654 %}
12655
12656 // This pattern is automatically generated from aarch64_ad.m4
12657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12658 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12659 %{
12660 match(Set dst (MinI imm src));
12661 ins_cost(INSN_COST * 3);
12662 expand %{
12663 rFlagsReg cr;
12664 compI_reg_imm0(cr, src);
12665 cmovI_reg_immM1_lt(dst, src, cr);
12666 %}
12667 %}
12668
12669 // This pattern is automatically generated from aarch64_ad.m4
12670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12671 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12672 %{
12673 match(Set dst (MaxI src imm));
12674 ins_cost(INSN_COST * 3);
12675 expand %{
12676 rFlagsReg cr;
12677 compI_reg_imm0(cr, src);
12678 cmovI_reg_imm0_gt(dst, src, cr);
12679 %}
12680 %}
12681
12682 // This pattern is automatically generated from aarch64_ad.m4
12683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12684 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12685 %{
12686 match(Set dst (MaxI imm src));
12687 ins_cost(INSN_COST * 3);
12688 expand %{
12689 rFlagsReg cr;
12690 compI_reg_imm0(cr, src);
12691 cmovI_reg_imm0_gt(dst, src, cr);
12692 %}
12693 %}
12694
12695 // This pattern is automatically generated from aarch64_ad.m4
12696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12697 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12698 %{
12699 match(Set dst (MaxI src imm));
12700 ins_cost(INSN_COST * 3);
12701 expand %{
12702 rFlagsReg cr;
12703 compI_reg_imm0(cr, src);
12704 cmovI_reg_imm1_gt(dst, src, cr);
12705 %}
12706 %}
12707
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12711 %{
12712 match(Set dst (MaxI imm src));
12713 ins_cost(INSN_COST * 3);
12714 expand %{
12715 rFlagsReg cr;
12716 compI_reg_imm0(cr, src);
12717 cmovI_reg_imm1_gt(dst, src, cr);
12718 %}
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12724 %{
12725 match(Set dst (MaxI src imm));
12726 ins_cost(INSN_COST * 3);
12727 expand %{
12728 rFlagsReg cr;
12729 compI_reg_imm0(cr, src);
12730 cmovI_reg_immM1_ge(dst, src, cr);
12731 %}
12732 %}
12733
12734 // This pattern is automatically generated from aarch64_ad.m4
12735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12736 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12737 %{
12738 match(Set dst (MaxI imm src));
12739 ins_cost(INSN_COST * 3);
12740 expand %{
12741 rFlagsReg cr;
12742 compI_reg_imm0(cr, src);
12743 cmovI_reg_immM1_ge(dst, src, cr);
12744 %}
12745 %}
12746
12747 // This pattern is automatically generated from aarch64_ad.m4
12748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12749 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12750 %{
12751 match(Set dst (ReverseI src));
12752 ins_cost(INSN_COST);
12753 format %{ "rbitw $dst, $src" %}
12754 ins_encode %{
12755 __ rbitw($dst$$Register, $src$$Register);
12756 %}
12757 ins_pipe(ialu_reg);
12758 %}
12759
12760 // This pattern is automatically generated from aarch64_ad.m4
12761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12762 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12763 %{
12764 match(Set dst (ReverseL src));
12765 ins_cost(INSN_COST);
12766 format %{ "rbit $dst, $src" %}
12767 ins_encode %{
12768 __ rbit($dst$$Register, $src$$Register);
12769 %}
12770 ins_pipe(ialu_reg);
12771 %}
12772
12773
12774 // END This section of the file is automatically generated. Do not edit --------------
12775
12776
12777 // ============================================================================
12778 // Floating Point Arithmetic Instructions
12779
12780 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12781 match(Set dst (AddHF src1 src2));
12782 format %{ "faddh $dst, $src1, $src2" %}
12783 ins_encode %{
12784 __ faddh($dst$$FloatRegister,
12785 $src1$$FloatRegister,
12786 $src2$$FloatRegister);
12787 %}
12788 ins_pipe(fp_dop_reg_reg_s);
12789 %}
12790
12791 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12792 match(Set dst (AddF src1 src2));
12793
12794 ins_cost(INSN_COST * 5);
12795 format %{ "fadds $dst, $src1, $src2" %}
12796
12797 ins_encode %{
12798 __ fadds(as_FloatRegister($dst$$reg),
12799 as_FloatRegister($src1$$reg),
12800 as_FloatRegister($src2$$reg));
12801 %}
12802
12803 ins_pipe(fp_dop_reg_reg_s);
12804 %}
12805
12806 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12807 match(Set dst (AddD src1 src2));
12808
12809 ins_cost(INSN_COST * 5);
12810 format %{ "faddd $dst, $src1, $src2" %}
12811
12812 ins_encode %{
12813 __ faddd(as_FloatRegister($dst$$reg),
12814 as_FloatRegister($src1$$reg),
12815 as_FloatRegister($src2$$reg));
12816 %}
12817
12818 ins_pipe(fp_dop_reg_reg_d);
12819 %}
12820
12821 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12822 match(Set dst (SubHF src1 src2));
12823 format %{ "fsubh $dst, $src1, $src2" %}
12824 ins_encode %{
12825 __ fsubh($dst$$FloatRegister,
12826 $src1$$FloatRegister,
12827 $src2$$FloatRegister);
12828 %}
12829 ins_pipe(fp_dop_reg_reg_s);
12830 %}
12831
12832 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12833 match(Set dst (SubF src1 src2));
12834
12835 ins_cost(INSN_COST * 5);
12836 format %{ "fsubs $dst, $src1, $src2" %}
12837
12838 ins_encode %{
12839 __ fsubs(as_FloatRegister($dst$$reg),
12840 as_FloatRegister($src1$$reg),
12841 as_FloatRegister($src2$$reg));
12842 %}
12843
12844 ins_pipe(fp_dop_reg_reg_s);
12845 %}
12846
12847 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12848 match(Set dst (SubD src1 src2));
12849
12850 ins_cost(INSN_COST * 5);
12851 format %{ "fsubd $dst, $src1, $src2" %}
12852
12853 ins_encode %{
12854 __ fsubd(as_FloatRegister($dst$$reg),
12855 as_FloatRegister($src1$$reg),
12856 as_FloatRegister($src2$$reg));
12857 %}
12858
12859 ins_pipe(fp_dop_reg_reg_d);
12860 %}
12861
12862 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12863 match(Set dst (MulHF src1 src2));
12864 format %{ "fmulh $dst, $src1, $src2" %}
12865 ins_encode %{
12866 __ fmulh($dst$$FloatRegister,
12867 $src1$$FloatRegister,
12868 $src2$$FloatRegister);
12869 %}
12870 ins_pipe(fp_dop_reg_reg_s);
12871 %}
12872
12873 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12874 match(Set dst (MulF src1 src2));
12875
12876 ins_cost(INSN_COST * 6);
12877 format %{ "fmuls $dst, $src1, $src2" %}
12878
12879 ins_encode %{
12880 __ fmuls(as_FloatRegister($dst$$reg),
12881 as_FloatRegister($src1$$reg),
12882 as_FloatRegister($src2$$reg));
12883 %}
12884
12885 ins_pipe(fp_dop_reg_reg_s);
12886 %}
12887
12888 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12889 match(Set dst (MulD src1 src2));
12890
12891 ins_cost(INSN_COST * 6);
12892 format %{ "fmuld $dst, $src1, $src2" %}
12893
12894 ins_encode %{
12895 __ fmuld(as_FloatRegister($dst$$reg),
12896 as_FloatRegister($src1$$reg),
12897 as_FloatRegister($src2$$reg));
12898 %}
12899
12900 ins_pipe(fp_dop_reg_reg_d);
12901 %}
12902
12903 // src1 * src2 + src3 (half-precision float)
12904 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12905 match(Set dst (FmaHF src3 (Binary src1 src2)));
12906 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12907 ins_encode %{
12908 assert(UseFMA, "Needs FMA instructions support.");
12909 __ fmaddh($dst$$FloatRegister,
12910 $src1$$FloatRegister,
12911 $src2$$FloatRegister,
12912 $src3$$FloatRegister);
12913 %}
12914 ins_pipe(pipe_class_default);
12915 %}
12916
12917 // src1 * src2 + src3
12918 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12919 match(Set dst (FmaF src3 (Binary src1 src2)));
12920
12921 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12922
12923 ins_encode %{
12924 assert(UseFMA, "Needs FMA instructions support.");
12925 __ fmadds(as_FloatRegister($dst$$reg),
12926 as_FloatRegister($src1$$reg),
12927 as_FloatRegister($src2$$reg),
12928 as_FloatRegister($src3$$reg));
12929 %}
12930
12931 ins_pipe(pipe_class_default);
12932 %}
12933
12934 // src1 * src2 + src3
12935 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12936 match(Set dst (FmaD src3 (Binary src1 src2)));
12937
12938 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12939
12940 ins_encode %{
12941 assert(UseFMA, "Needs FMA instructions support.");
12942 __ fmaddd(as_FloatRegister($dst$$reg),
12943 as_FloatRegister($src1$$reg),
12944 as_FloatRegister($src2$$reg),
12945 as_FloatRegister($src3$$reg));
12946 %}
12947
12948 ins_pipe(pipe_class_default);
12949 %}
12950
12951 // src1 * (-src2) + src3
12952 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12953 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12954 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12955
12956 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12957
12958 ins_encode %{
12959 assert(UseFMA, "Needs FMA instructions support.");
12960 __ fmsubs(as_FloatRegister($dst$$reg),
12961 as_FloatRegister($src1$$reg),
12962 as_FloatRegister($src2$$reg),
12963 as_FloatRegister($src3$$reg));
12964 %}
12965
12966 ins_pipe(pipe_class_default);
12967 %}
12968
12969 // src1 * (-src2) + src3
12970 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12971 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12972 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12973
12974 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12975
12976 ins_encode %{
12977 assert(UseFMA, "Needs FMA instructions support.");
12978 __ fmsubd(as_FloatRegister($dst$$reg),
12979 as_FloatRegister($src1$$reg),
12980 as_FloatRegister($src2$$reg),
12981 as_FloatRegister($src3$$reg));
12982 %}
12983
12984 ins_pipe(pipe_class_default);
12985 %}
12986
12987 // src1 * (-src2) - src3
12988 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12989 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12990 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12991
12992 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
12993
12994 ins_encode %{
12995 assert(UseFMA, "Needs FMA instructions support.");
12996 __ fnmadds(as_FloatRegister($dst$$reg),
12997 as_FloatRegister($src1$$reg),
12998 as_FloatRegister($src2$$reg),
12999 as_FloatRegister($src3$$reg));
13000 %}
13001
13002 ins_pipe(pipe_class_default);
13003 %}
13004
13005 // src1 * (-src2) - src3
13006 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13007 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13008 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13009
13010 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13011
13012 ins_encode %{
13013 assert(UseFMA, "Needs FMA instructions support.");
13014 __ fnmaddd(as_FloatRegister($dst$$reg),
13015 as_FloatRegister($src1$$reg),
13016 as_FloatRegister($src2$$reg),
13017 as_FloatRegister($src3$$reg));
13018 %}
13019
13020 ins_pipe(pipe_class_default);
13021 %}
13022
13023 // src1 * src2 - src3
13024 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13025 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13026
13027 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13028
13029 ins_encode %{
13030 assert(UseFMA, "Needs FMA instructions support.");
13031 __ fnmsubs(as_FloatRegister($dst$$reg),
13032 as_FloatRegister($src1$$reg),
13033 as_FloatRegister($src2$$reg),
13034 as_FloatRegister($src3$$reg));
13035 %}
13036
13037 ins_pipe(pipe_class_default);
13038 %}
13039
13040 // src1 * src2 - src3
13041 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13042 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13043
13044 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13045
13046 ins_encode %{
13047 assert(UseFMA, "Needs FMA instructions support.");
13048 // n.b. insn name should be fnmsubd
13049 __ fnmsub(as_FloatRegister($dst$$reg),
13050 as_FloatRegister($src1$$reg),
13051 as_FloatRegister($src2$$reg),
13052 as_FloatRegister($src3$$reg));
13053 %}
13054
13055 ins_pipe(pipe_class_default);
13056 %}
13057
13058 // Math.max(HH)H (half-precision float)
13059 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13060 match(Set dst (MaxHF src1 src2));
13061 format %{ "fmaxh $dst, $src1, $src2" %}
13062 ins_encode %{
13063 __ fmaxh($dst$$FloatRegister,
13064 $src1$$FloatRegister,
13065 $src2$$FloatRegister);
13066 %}
13067 ins_pipe(fp_dop_reg_reg_s);
13068 %}
13069
13070 // Math.min(HH)H (half-precision float)
13071 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13072 match(Set dst (MinHF src1 src2));
13073 format %{ "fminh $dst, $src1, $src2" %}
13074 ins_encode %{
13075 __ fminh($dst$$FloatRegister,
13076 $src1$$FloatRegister,
13077 $src2$$FloatRegister);
13078 %}
13079 ins_pipe(fp_dop_reg_reg_s);
13080 %}
13081
13082 // Math.max(FF)F
13083 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13084 match(Set dst (MaxF src1 src2));
13085
13086 format %{ "fmaxs $dst, $src1, $src2" %}
13087 ins_encode %{
13088 __ fmaxs(as_FloatRegister($dst$$reg),
13089 as_FloatRegister($src1$$reg),
13090 as_FloatRegister($src2$$reg));
13091 %}
13092
13093 ins_pipe(fp_dop_reg_reg_s);
13094 %}
13095
13096 // Math.min(FF)F
13097 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13098 match(Set dst (MinF src1 src2));
13099
13100 format %{ "fmins $dst, $src1, $src2" %}
13101 ins_encode %{
13102 __ fmins(as_FloatRegister($dst$$reg),
13103 as_FloatRegister($src1$$reg),
13104 as_FloatRegister($src2$$reg));
13105 %}
13106
13107 ins_pipe(fp_dop_reg_reg_s);
13108 %}
13109
13110 // Math.max(DD)D
13111 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13112 match(Set dst (MaxD src1 src2));
13113
13114 format %{ "fmaxd $dst, $src1, $src2" %}
13115 ins_encode %{
13116 __ fmaxd(as_FloatRegister($dst$$reg),
13117 as_FloatRegister($src1$$reg),
13118 as_FloatRegister($src2$$reg));
13119 %}
13120
13121 ins_pipe(fp_dop_reg_reg_d);
13122 %}
13123
13124 // Math.min(DD)D
13125 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13126 match(Set dst (MinD src1 src2));
13127
13128 format %{ "fmind $dst, $src1, $src2" %}
13129 ins_encode %{
13130 __ fmind(as_FloatRegister($dst$$reg),
13131 as_FloatRegister($src1$$reg),
13132 as_FloatRegister($src2$$reg));
13133 %}
13134
13135 ins_pipe(fp_dop_reg_reg_d);
13136 %}
13137
13138 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13139 match(Set dst (DivHF src1 src2));
13140 format %{ "fdivh $dst, $src1, $src2" %}
13141 ins_encode %{
13142 __ fdivh($dst$$FloatRegister,
13143 $src1$$FloatRegister,
13144 $src2$$FloatRegister);
13145 %}
13146 ins_pipe(fp_div_s);
13147 %}
13148
13149 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13150 match(Set dst (DivF src1 src2));
13151
13152 ins_cost(INSN_COST * 18);
13153 format %{ "fdivs $dst, $src1, $src2" %}
13154
13155 ins_encode %{
13156 __ fdivs(as_FloatRegister($dst$$reg),
13157 as_FloatRegister($src1$$reg),
13158 as_FloatRegister($src2$$reg));
13159 %}
13160
13161 ins_pipe(fp_div_s);
13162 %}
13163
13164 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13165 match(Set dst (DivD src1 src2));
13166
13167 ins_cost(INSN_COST * 32);
13168 format %{ "fdivd $dst, $src1, $src2" %}
13169
13170 ins_encode %{
13171 __ fdivd(as_FloatRegister($dst$$reg),
13172 as_FloatRegister($src1$$reg),
13173 as_FloatRegister($src2$$reg));
13174 %}
13175
13176 ins_pipe(fp_div_d);
13177 %}
13178
13179 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13180 match(Set dst (NegF src));
13181
13182 ins_cost(INSN_COST * 3);
13183 format %{ "fneg $dst, $src" %}
13184
13185 ins_encode %{
13186 __ fnegs(as_FloatRegister($dst$$reg),
13187 as_FloatRegister($src$$reg));
13188 %}
13189
13190 ins_pipe(fp_uop_s);
13191 %}
13192
13193 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13194 match(Set dst (NegD src));
13195
13196 ins_cost(INSN_COST * 3);
13197 format %{ "fnegd $dst, $src" %}
13198
13199 ins_encode %{
13200 __ fnegd(as_FloatRegister($dst$$reg),
13201 as_FloatRegister($src$$reg));
13202 %}
13203
13204 ins_pipe(fp_uop_d);
13205 %}
13206
13207 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13208 %{
13209 match(Set dst (AbsI src));
13210
13211 effect(KILL cr);
13212 ins_cost(INSN_COST * 2);
13213 format %{ "cmpw $src, zr\n\t"
13214 "cnegw $dst, $src, Assembler::LT\t# int abs"
13215 %}
13216
13217 ins_encode %{
13218 __ cmpw(as_Register($src$$reg), zr);
13219 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13220 %}
13221 ins_pipe(pipe_class_default);
13222 %}
13223
13224 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13225 %{
13226 match(Set dst (AbsL src));
13227
13228 effect(KILL cr);
13229 ins_cost(INSN_COST * 2);
13230 format %{ "cmp $src, zr\n\t"
13231 "cneg $dst, $src, Assembler::LT\t# long abs"
13232 %}
13233
13234 ins_encode %{
13235 __ cmp(as_Register($src$$reg), zr);
13236 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13237 %}
13238 ins_pipe(pipe_class_default);
13239 %}
13240
13241 instruct absF_reg(vRegF dst, vRegF src) %{
13242 match(Set dst (AbsF src));
13243
13244 ins_cost(INSN_COST * 3);
13245 format %{ "fabss $dst, $src" %}
13246 ins_encode %{
13247 __ fabss(as_FloatRegister($dst$$reg),
13248 as_FloatRegister($src$$reg));
13249 %}
13250
13251 ins_pipe(fp_uop_s);
13252 %}
13253
13254 instruct absD_reg(vRegD dst, vRegD src) %{
13255 match(Set dst (AbsD src));
13256
13257 ins_cost(INSN_COST * 3);
13258 format %{ "fabsd $dst, $src" %}
13259 ins_encode %{
13260 __ fabsd(as_FloatRegister($dst$$reg),
13261 as_FloatRegister($src$$reg));
13262 %}
13263
13264 ins_pipe(fp_uop_d);
13265 %}
13266
13267 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13268 match(Set dst (AbsF (SubF src1 src2)));
13269
13270 ins_cost(INSN_COST * 3);
13271 format %{ "fabds $dst, $src1, $src2" %}
13272 ins_encode %{
13273 __ fabds(as_FloatRegister($dst$$reg),
13274 as_FloatRegister($src1$$reg),
13275 as_FloatRegister($src2$$reg));
13276 %}
13277
13278 ins_pipe(fp_uop_s);
13279 %}
13280
13281 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13282 match(Set dst (AbsD (SubD src1 src2)));
13283
13284 ins_cost(INSN_COST * 3);
13285 format %{ "fabdd $dst, $src1, $src2" %}
13286 ins_encode %{
13287 __ fabdd(as_FloatRegister($dst$$reg),
13288 as_FloatRegister($src1$$reg),
13289 as_FloatRegister($src2$$reg));
13290 %}
13291
13292 ins_pipe(fp_uop_d);
13293 %}
13294
13295 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13296 match(Set dst (SqrtD src));
13297
13298 ins_cost(INSN_COST * 50);
13299 format %{ "fsqrtd $dst, $src" %}
13300 ins_encode %{
13301 __ fsqrtd(as_FloatRegister($dst$$reg),
13302 as_FloatRegister($src$$reg));
13303 %}
13304
13305 ins_pipe(fp_div_s);
13306 %}
13307
13308 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13309 match(Set dst (SqrtF src));
13310
13311 ins_cost(INSN_COST * 50);
13312 format %{ "fsqrts $dst, $src" %}
13313 ins_encode %{
13314 __ fsqrts(as_FloatRegister($dst$$reg),
13315 as_FloatRegister($src$$reg));
13316 %}
13317
13318 ins_pipe(fp_div_d);
13319 %}
13320
13321 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13322 match(Set dst (SqrtHF src));
13323 format %{ "fsqrth $dst, $src" %}
13324 ins_encode %{
13325 __ fsqrth($dst$$FloatRegister,
13326 $src$$FloatRegister);
13327 %}
13328 ins_pipe(fp_div_s);
13329 %}
13330
13331 // Math.rint, floor, ceil
13332 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13333 match(Set dst (RoundDoubleMode src rmode));
13334 format %{ "frint $dst, $src, $rmode" %}
13335 ins_encode %{
13336 switch ($rmode$$constant) {
13337 case RoundDoubleModeNode::rmode_rint:
13338 __ frintnd(as_FloatRegister($dst$$reg),
13339 as_FloatRegister($src$$reg));
13340 break;
13341 case RoundDoubleModeNode::rmode_floor:
13342 __ frintmd(as_FloatRegister($dst$$reg),
13343 as_FloatRegister($src$$reg));
13344 break;
13345 case RoundDoubleModeNode::rmode_ceil:
13346 __ frintpd(as_FloatRegister($dst$$reg),
13347 as_FloatRegister($src$$reg));
13348 break;
13349 }
13350 %}
13351 ins_pipe(fp_uop_d);
13352 %}
13353
13354 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13355 match(Set dst (CopySignD src1 (Binary src2 zero)));
13356 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13357 format %{ "CopySignD $dst $src1 $src2" %}
13358 ins_encode %{
13359 FloatRegister dst = as_FloatRegister($dst$$reg),
13360 src1 = as_FloatRegister($src1$$reg),
13361 src2 = as_FloatRegister($src2$$reg),
13362 zero = as_FloatRegister($zero$$reg);
13363 __ fnegd(dst, zero);
13364 __ bsl(dst, __ T8B, src2, src1);
13365 %}
13366 ins_pipe(fp_uop_d);
13367 %}
13368
13369 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13370 match(Set dst (CopySignF src1 src2));
13371 effect(TEMP_DEF dst, USE src1, USE src2);
13372 format %{ "CopySignF $dst $src1 $src2" %}
13373 ins_encode %{
13374 FloatRegister dst = as_FloatRegister($dst$$reg),
13375 src1 = as_FloatRegister($src1$$reg),
13376 src2 = as_FloatRegister($src2$$reg);
13377 __ movi(dst, __ T2S, 0x80, 24);
13378 __ bsl(dst, __ T8B, src2, src1);
13379 %}
13380 ins_pipe(fp_uop_d);
13381 %}
13382
13383 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13384 match(Set dst (SignumD src (Binary zero one)));
13385 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13386 format %{ "signumD $dst, $src" %}
13387 ins_encode %{
13388 FloatRegister src = as_FloatRegister($src$$reg),
13389 dst = as_FloatRegister($dst$$reg),
13390 zero = as_FloatRegister($zero$$reg),
13391 one = as_FloatRegister($one$$reg);
13392 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13393 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13394 // Bit selection instruction gets bit from "one" for each enabled bit in
13395 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13396 // NaN the whole "src" will be copied because "dst" is zero. For all other
13397 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13398 // from "src", and all other bits are copied from 1.0.
13399 __ bsl(dst, __ T8B, one, src);
13400 %}
13401 ins_pipe(fp_uop_d);
13402 %}
13403
13404 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13405 match(Set dst (SignumF src (Binary zero one)));
13406 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13407 format %{ "signumF $dst, $src" %}
13408 ins_encode %{
13409 FloatRegister src = as_FloatRegister($src$$reg),
13410 dst = as_FloatRegister($dst$$reg),
13411 zero = as_FloatRegister($zero$$reg),
13412 one = as_FloatRegister($one$$reg);
13413 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13414 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13415 // Bit selection instruction gets bit from "one" for each enabled bit in
13416 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13417 // NaN the whole "src" will be copied because "dst" is zero. For all other
13418 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13419 // from "src", and all other bits are copied from 1.0.
13420 __ bsl(dst, __ T8B, one, src);
13421 %}
13422 ins_pipe(fp_uop_d);
13423 %}
13424
13425 instruct onspinwait() %{
13426 match(OnSpinWait);
13427 ins_cost(INSN_COST);
13428
13429 format %{ "onspinwait" %}
13430
13431 ins_encode %{
13432 __ spin_wait();
13433 %}
13434 ins_pipe(pipe_class_empty);
13435 %}
13436
13437 // ============================================================================
13438 // Logical Instructions
13439
13440 // Integer Logical Instructions
13441
13442 // And Instructions
13443
13444
13445 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13446 match(Set dst (AndI src1 src2));
13447
13448 format %{ "andw $dst, $src1, $src2\t# int" %}
13449
13450 ins_cost(INSN_COST);
13451 ins_encode %{
13452 __ andw(as_Register($dst$$reg),
13453 as_Register($src1$$reg),
13454 as_Register($src2$$reg));
13455 %}
13456
13457 ins_pipe(ialu_reg_reg);
13458 %}
13459
13460 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13461 match(Set dst (AndI src1 src2));
13462
13463 format %{ "andsw $dst, $src1, $src2\t# int" %}
13464
13465 ins_cost(INSN_COST);
13466 ins_encode %{
13467 __ andw(as_Register($dst$$reg),
13468 as_Register($src1$$reg),
13469 (uint64_t)($src2$$constant));
13470 %}
13471
13472 ins_pipe(ialu_reg_imm);
13473 %}
13474
13475 // Or Instructions
13476
13477 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13478 match(Set dst (OrI src1 src2));
13479
13480 format %{ "orrw $dst, $src1, $src2\t# int" %}
13481
13482 ins_cost(INSN_COST);
13483 ins_encode %{
13484 __ orrw(as_Register($dst$$reg),
13485 as_Register($src1$$reg),
13486 as_Register($src2$$reg));
13487 %}
13488
13489 ins_pipe(ialu_reg_reg);
13490 %}
13491
13492 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13493 match(Set dst (OrI src1 src2));
13494
13495 format %{ "orrw $dst, $src1, $src2\t# int" %}
13496
13497 ins_cost(INSN_COST);
13498 ins_encode %{
13499 __ orrw(as_Register($dst$$reg),
13500 as_Register($src1$$reg),
13501 (uint64_t)($src2$$constant));
13502 %}
13503
13504 ins_pipe(ialu_reg_imm);
13505 %}
13506
13507 // Xor Instructions
13508
13509 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13510 match(Set dst (XorI src1 src2));
13511
13512 format %{ "eorw $dst, $src1, $src2\t# int" %}
13513
13514 ins_cost(INSN_COST);
13515 ins_encode %{
13516 __ eorw(as_Register($dst$$reg),
13517 as_Register($src1$$reg),
13518 as_Register($src2$$reg));
13519 %}
13520
13521 ins_pipe(ialu_reg_reg);
13522 %}
13523
13524 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13525 match(Set dst (XorI src1 src2));
13526
13527 format %{ "eorw $dst, $src1, $src2\t# int" %}
13528
13529 ins_cost(INSN_COST);
13530 ins_encode %{
13531 __ eorw(as_Register($dst$$reg),
13532 as_Register($src1$$reg),
13533 (uint64_t)($src2$$constant));
13534 %}
13535
13536 ins_pipe(ialu_reg_imm);
13537 %}
13538
13539 // Long Logical Instructions
13540 // TODO
13541
13542 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13543 match(Set dst (AndL src1 src2));
13544
13545 format %{ "and $dst, $src1, $src2\t# int" %}
13546
13547 ins_cost(INSN_COST);
13548 ins_encode %{
13549 __ andr(as_Register($dst$$reg),
13550 as_Register($src1$$reg),
13551 as_Register($src2$$reg));
13552 %}
13553
13554 ins_pipe(ialu_reg_reg);
13555 %}
13556
13557 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13558 match(Set dst (AndL src1 src2));
13559
13560 format %{ "and $dst, $src1, $src2\t# int" %}
13561
13562 ins_cost(INSN_COST);
13563 ins_encode %{
13564 __ andr(as_Register($dst$$reg),
13565 as_Register($src1$$reg),
13566 (uint64_t)($src2$$constant));
13567 %}
13568
13569 ins_pipe(ialu_reg_imm);
13570 %}
13571
13572 // Or Instructions
13573
13574 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13575 match(Set dst (OrL src1 src2));
13576
13577 format %{ "orr $dst, $src1, $src2\t# int" %}
13578
13579 ins_cost(INSN_COST);
13580 ins_encode %{
13581 __ orr(as_Register($dst$$reg),
13582 as_Register($src1$$reg),
13583 as_Register($src2$$reg));
13584 %}
13585
13586 ins_pipe(ialu_reg_reg);
13587 %}
13588
13589 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13590 match(Set dst (OrL src1 src2));
13591
13592 format %{ "orr $dst, $src1, $src2\t# int" %}
13593
13594 ins_cost(INSN_COST);
13595 ins_encode %{
13596 __ orr(as_Register($dst$$reg),
13597 as_Register($src1$$reg),
13598 (uint64_t)($src2$$constant));
13599 %}
13600
13601 ins_pipe(ialu_reg_imm);
13602 %}
13603
13604 // Xor Instructions
13605
13606 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13607 match(Set dst (XorL src1 src2));
13608
13609 format %{ "eor $dst, $src1, $src2\t# int" %}
13610
13611 ins_cost(INSN_COST);
13612 ins_encode %{
13613 __ eor(as_Register($dst$$reg),
13614 as_Register($src1$$reg),
13615 as_Register($src2$$reg));
13616 %}
13617
13618 ins_pipe(ialu_reg_reg);
13619 %}
13620
13621 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13622 match(Set dst (XorL src1 src2));
13623
13624 ins_cost(INSN_COST);
13625 format %{ "eor $dst, $src1, $src2\t# int" %}
13626
13627 ins_encode %{
13628 __ eor(as_Register($dst$$reg),
13629 as_Register($src1$$reg),
13630 (uint64_t)($src2$$constant));
13631 %}
13632
13633 ins_pipe(ialu_reg_imm);
13634 %}
13635
13636 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13637 %{
13638 match(Set dst (ConvI2L src));
13639
13640 ins_cost(INSN_COST);
13641 format %{ "sxtw $dst, $src\t# i2l" %}
13642 ins_encode %{
13643 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13644 %}
13645 ins_pipe(ialu_reg_shift);
13646 %}
13647
13648 // this pattern occurs in bigmath arithmetic
13649 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13650 %{
13651 match(Set dst (AndL (ConvI2L src) mask));
13652
13653 ins_cost(INSN_COST);
13654 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13655 ins_encode %{
13656 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13657 %}
13658
13659 ins_pipe(ialu_reg_shift);
13660 %}
13661
13662 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13663 match(Set dst (ConvL2I src));
13664
13665 ins_cost(INSN_COST);
13666 format %{ "movw $dst, $src \t// l2i" %}
13667
13668 ins_encode %{
13669 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13670 %}
13671
13672 ins_pipe(ialu_reg);
13673 %}
13674
13675 instruct convD2F_reg(vRegF dst, vRegD src) %{
13676 match(Set dst (ConvD2F src));
13677
13678 ins_cost(INSN_COST * 5);
13679 format %{ "fcvtd $dst, $src \t// d2f" %}
13680
13681 ins_encode %{
13682 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13683 %}
13684
13685 ins_pipe(fp_d2f);
13686 %}
13687
13688 instruct convF2D_reg(vRegD dst, vRegF src) %{
13689 match(Set dst (ConvF2D src));
13690
13691 ins_cost(INSN_COST * 5);
13692 format %{ "fcvts $dst, $src \t// f2d" %}
13693
13694 ins_encode %{
13695 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13696 %}
13697
13698 ins_pipe(fp_f2d);
13699 %}
13700
13701 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13702 match(Set dst (ConvF2I src));
13703
13704 ins_cost(INSN_COST * 5);
13705 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13706
13707 ins_encode %{
13708 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13709 %}
13710
13711 ins_pipe(fp_f2i);
13712 %}
13713
13714 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13715 match(Set dst (ConvF2L src));
13716
13717 ins_cost(INSN_COST * 5);
13718 format %{ "fcvtzs $dst, $src \t// f2l" %}
13719
13720 ins_encode %{
13721 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13722 %}
13723
13724 ins_pipe(fp_f2l);
13725 %}
13726
13727 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13728 match(Set dst (ConvF2HF src));
13729 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13730 "smov $dst, $tmp\t# move result from $tmp to $dst"
13731 %}
13732 effect(TEMP tmp);
13733 ins_encode %{
13734 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13735 %}
13736 ins_pipe(pipe_slow);
13737 %}
13738
13739 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13740 match(Set dst (ConvHF2F src));
13741 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13742 "fcvt $dst, $tmp\t# convert half to single precision"
13743 %}
13744 effect(TEMP tmp);
13745 ins_encode %{
13746 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13747 %}
13748 ins_pipe(pipe_slow);
13749 %}
13750
13751 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13752 match(Set dst (ConvI2F src));
13753
13754 ins_cost(INSN_COST * 5);
13755 format %{ "scvtfws $dst, $src \t// i2f" %}
13756
13757 ins_encode %{
13758 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13759 %}
13760
13761 ins_pipe(fp_i2f);
13762 %}
13763
13764 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13765 match(Set dst (ConvL2F src));
13766
13767 ins_cost(INSN_COST * 5);
13768 format %{ "scvtfs $dst, $src \t// l2f" %}
13769
13770 ins_encode %{
13771 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13772 %}
13773
13774 ins_pipe(fp_l2f);
13775 %}
13776
13777 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13778 match(Set dst (ConvD2I src));
13779
13780 ins_cost(INSN_COST * 5);
13781 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13782
13783 ins_encode %{
13784 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13785 %}
13786
13787 ins_pipe(fp_d2i);
13788 %}
13789
13790 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13791 match(Set dst (ConvD2L src));
13792
13793 ins_cost(INSN_COST * 5);
13794 format %{ "fcvtzd $dst, $src \t// d2l" %}
13795
13796 ins_encode %{
13797 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13798 %}
13799
13800 ins_pipe(fp_d2l);
13801 %}
13802
13803 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13804 match(Set dst (ConvI2D src));
13805
13806 ins_cost(INSN_COST * 5);
13807 format %{ "scvtfwd $dst, $src \t// i2d" %}
13808
13809 ins_encode %{
13810 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13811 %}
13812
13813 ins_pipe(fp_i2d);
13814 %}
13815
13816 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13817 match(Set dst (ConvL2D src));
13818
13819 ins_cost(INSN_COST * 5);
13820 format %{ "scvtfd $dst, $src \t// l2d" %}
13821
13822 ins_encode %{
13823 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13824 %}
13825
13826 ins_pipe(fp_l2d);
13827 %}
13828
13829 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13830 %{
13831 match(Set dst (RoundD src));
13832 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13833 format %{ "java_round_double $dst,$src"%}
13834 ins_encode %{
13835 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13836 as_FloatRegister($ftmp$$reg));
13837 %}
13838 ins_pipe(pipe_slow);
13839 %}
13840
13841 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13842 %{
13843 match(Set dst (RoundF src));
13844 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13845 format %{ "java_round_float $dst,$src"%}
13846 ins_encode %{
13847 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13848 as_FloatRegister($ftmp$$reg));
13849 %}
13850 ins_pipe(pipe_slow);
13851 %}
13852
13853 // stack <-> reg and reg <-> reg shuffles with no conversion
13854
13855 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13856
13857 match(Set dst (MoveF2I src));
13858
13859 effect(DEF dst, USE src);
13860
13861 ins_cost(4 * INSN_COST);
13862
13863 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13864
13865 ins_encode %{
13866 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13867 %}
13868
13869 ins_pipe(iload_reg_reg);
13870
13871 %}
13872
13873 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13874
13875 match(Set dst (MoveI2F src));
13876
13877 effect(DEF dst, USE src);
13878
13879 ins_cost(4 * INSN_COST);
13880
13881 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13882
13883 ins_encode %{
13884 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13885 %}
13886
13887 ins_pipe(pipe_class_memory);
13888
13889 %}
13890
13891 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13892
13893 match(Set dst (MoveD2L src));
13894
13895 effect(DEF dst, USE src);
13896
13897 ins_cost(4 * INSN_COST);
13898
13899 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13900
13901 ins_encode %{
13902 __ ldr($dst$$Register, Address(sp, $src$$disp));
13903 %}
13904
13905 ins_pipe(iload_reg_reg);
13906
13907 %}
13908
13909 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13910
13911 match(Set dst (MoveL2D src));
13912
13913 effect(DEF dst, USE src);
13914
13915 ins_cost(4 * INSN_COST);
13916
13917 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13918
13919 ins_encode %{
13920 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13921 %}
13922
13923 ins_pipe(pipe_class_memory);
13924
13925 %}
13926
13927 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13928
13929 match(Set dst (MoveF2I src));
13930
13931 effect(DEF dst, USE src);
13932
13933 ins_cost(INSN_COST);
13934
13935 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13936
13937 ins_encode %{
13938 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13939 %}
13940
13941 ins_pipe(pipe_class_memory);
13942
13943 %}
13944
13945 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13946
13947 match(Set dst (MoveI2F src));
13948
13949 effect(DEF dst, USE src);
13950
13951 ins_cost(INSN_COST);
13952
13953 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13954
13955 ins_encode %{
13956 __ strw($src$$Register, Address(sp, $dst$$disp));
13957 %}
13958
13959 ins_pipe(istore_reg_reg);
13960
13961 %}
13962
13963 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13964
13965 match(Set dst (MoveD2L src));
13966
13967 effect(DEF dst, USE src);
13968
13969 ins_cost(INSN_COST);
13970
13971 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13972
13973 ins_encode %{
13974 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13975 %}
13976
13977 ins_pipe(pipe_class_memory);
13978
13979 %}
13980
13981 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13982
13983 match(Set dst (MoveL2D src));
13984
13985 effect(DEF dst, USE src);
13986
13987 ins_cost(INSN_COST);
13988
13989 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13990
13991 ins_encode %{
13992 __ str($src$$Register, Address(sp, $dst$$disp));
13993 %}
13994
13995 ins_pipe(istore_reg_reg);
13996
13997 %}
13998
13999 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14000
14001 match(Set dst (MoveF2I src));
14002
14003 effect(DEF dst, USE src);
14004
14005 ins_cost(INSN_COST);
14006
14007 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14008
14009 ins_encode %{
14010 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14011 %}
14012
14013 ins_pipe(fp_f2i);
14014
14015 %}
14016
14017 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14018
14019 match(Set dst (MoveI2F src));
14020
14021 effect(DEF dst, USE src);
14022
14023 ins_cost(INSN_COST);
14024
14025 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14026
14027 ins_encode %{
14028 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14029 %}
14030
14031 ins_pipe(fp_i2f);
14032
14033 %}
14034
14035 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14036
14037 match(Set dst (MoveD2L src));
14038
14039 effect(DEF dst, USE src);
14040
14041 ins_cost(INSN_COST);
14042
14043 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14044
14045 ins_encode %{
14046 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14047 %}
14048
14049 ins_pipe(fp_d2l);
14050
14051 %}
14052
14053 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14054
14055 match(Set dst (MoveL2D src));
14056
14057 effect(DEF dst, USE src);
14058
14059 ins_cost(INSN_COST);
14060
14061 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14062
14063 ins_encode %{
14064 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14065 %}
14066
14067 ins_pipe(fp_l2d);
14068
14069 %}
14070
14071 // ============================================================================
14072 // clearing of an array
14073
14074 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14075 %{
14076 match(Set dummy (ClearArray cnt base));
14077 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14078
14079 ins_cost(4 * INSN_COST);
14080 format %{ "ClearArray $cnt, $base" %}
14081
14082 ins_encode %{
14083 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14084 if (tpc == nullptr) {
14085 ciEnv::current()->record_failure("CodeCache is full");
14086 return;
14087 }
14088 %}
14089
14090 ins_pipe(pipe_class_memory);
14091 %}
14092
14093 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14094 %{
14095 predicate((uint64_t)n->in(2)->get_long()
14096 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14097 match(Set dummy (ClearArray cnt base));
14098 effect(TEMP temp, USE_KILL base, KILL cr);
14099
14100 ins_cost(4 * INSN_COST);
14101 format %{ "ClearArray $cnt, $base" %}
14102
14103 ins_encode %{
14104 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14105 if (tpc == nullptr) {
14106 ciEnv::current()->record_failure("CodeCache is full");
14107 return;
14108 }
14109 %}
14110
14111 ins_pipe(pipe_class_memory);
14112 %}
14113
14114 // ============================================================================
14115 // Overflow Math Instructions
14116
14117 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14118 %{
14119 match(Set cr (OverflowAddI op1 op2));
14120
14121 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14122 ins_cost(INSN_COST);
14123 ins_encode %{
14124 __ cmnw($op1$$Register, $op2$$Register);
14125 %}
14126
14127 ins_pipe(icmp_reg_reg);
14128 %}
14129
14130 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14131 %{
14132 match(Set cr (OverflowAddI op1 op2));
14133
14134 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14135 ins_cost(INSN_COST);
14136 ins_encode %{
14137 __ cmnw($op1$$Register, $op2$$constant);
14138 %}
14139
14140 ins_pipe(icmp_reg_imm);
14141 %}
14142
14143 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14144 %{
14145 match(Set cr (OverflowAddL op1 op2));
14146
14147 format %{ "cmn $op1, $op2\t# overflow check long" %}
14148 ins_cost(INSN_COST);
14149 ins_encode %{
14150 __ cmn($op1$$Register, $op2$$Register);
14151 %}
14152
14153 ins_pipe(icmp_reg_reg);
14154 %}
14155
14156 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14157 %{
14158 match(Set cr (OverflowAddL op1 op2));
14159
14160 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14161 ins_cost(INSN_COST);
14162 ins_encode %{
14163 __ adds(zr, $op1$$Register, $op2$$constant);
14164 %}
14165
14166 ins_pipe(icmp_reg_imm);
14167 %}
14168
14169 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14170 %{
14171 match(Set cr (OverflowSubI op1 op2));
14172
14173 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14174 ins_cost(INSN_COST);
14175 ins_encode %{
14176 __ cmpw($op1$$Register, $op2$$Register);
14177 %}
14178
14179 ins_pipe(icmp_reg_reg);
14180 %}
14181
14182 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14183 %{
14184 match(Set cr (OverflowSubI op1 op2));
14185
14186 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14187 ins_cost(INSN_COST);
14188 ins_encode %{
14189 __ cmpw($op1$$Register, $op2$$constant);
14190 %}
14191
14192 ins_pipe(icmp_reg_imm);
14193 %}
14194
14195 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14196 %{
14197 match(Set cr (OverflowSubL op1 op2));
14198
14199 format %{ "cmp $op1, $op2\t# overflow check long" %}
14200 ins_cost(INSN_COST);
14201 ins_encode %{
14202 __ cmp($op1$$Register, $op2$$Register);
14203 %}
14204
14205 ins_pipe(icmp_reg_reg);
14206 %}
14207
14208 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14209 %{
14210 match(Set cr (OverflowSubL op1 op2));
14211
14212 format %{ "cmp $op1, $op2\t# overflow check long" %}
14213 ins_cost(INSN_COST);
14214 ins_encode %{
14215 __ subs(zr, $op1$$Register, $op2$$constant);
14216 %}
14217
14218 ins_pipe(icmp_reg_imm);
14219 %}
14220
14221 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14222 %{
14223 match(Set cr (OverflowSubI zero op1));
14224
14225 format %{ "cmpw zr, $op1\t# overflow check int" %}
14226 ins_cost(INSN_COST);
14227 ins_encode %{
14228 __ cmpw(zr, $op1$$Register);
14229 %}
14230
14231 ins_pipe(icmp_reg_imm);
14232 %}
14233
14234 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14235 %{
14236 match(Set cr (OverflowSubL zero op1));
14237
14238 format %{ "cmp zr, $op1\t# overflow check long" %}
14239 ins_cost(INSN_COST);
14240 ins_encode %{
14241 __ cmp(zr, $op1$$Register);
14242 %}
14243
14244 ins_pipe(icmp_reg_imm);
14245 %}
14246
14247 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14248 %{
14249 match(Set cr (OverflowMulI op1 op2));
14250
14251 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14252 "cmp rscratch1, rscratch1, sxtw\n\t"
14253 "movw rscratch1, #0x80000000\n\t"
14254 "cselw rscratch1, rscratch1, zr, NE\n\t"
14255 "cmpw rscratch1, #1" %}
14256 ins_cost(5 * INSN_COST);
14257 ins_encode %{
14258 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14259 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14260 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14261 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14262 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14263 %}
14264
14265 ins_pipe(pipe_slow);
14266 %}
14267
14268 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14269 %{
14270 match(If cmp (OverflowMulI op1 op2));
14271 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14272 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14273 effect(USE labl, KILL cr);
14274
14275 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14276 "cmp rscratch1, rscratch1, sxtw\n\t"
14277 "b$cmp $labl" %}
14278 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14279 ins_encode %{
14280 Label* L = $labl$$label;
14281 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14282 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14283 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14284 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14285 %}
14286
14287 ins_pipe(pipe_serial);
14288 %}
14289
14290 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14291 %{
14292 match(Set cr (OverflowMulL op1 op2));
14293
14294 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14295 "smulh rscratch2, $op1, $op2\n\t"
14296 "cmp rscratch2, rscratch1, ASR #63\n\t"
14297 "movw rscratch1, #0x80000000\n\t"
14298 "cselw rscratch1, rscratch1, zr, NE\n\t"
14299 "cmpw rscratch1, #1" %}
14300 ins_cost(6 * INSN_COST);
14301 ins_encode %{
14302 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14303 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14304 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14305 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14306 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14307 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14308 %}
14309
14310 ins_pipe(pipe_slow);
14311 %}
14312
14313 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14314 %{
14315 match(If cmp (OverflowMulL op1 op2));
14316 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14317 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14318 effect(USE labl, KILL cr);
14319
14320 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14321 "smulh rscratch2, $op1, $op2\n\t"
14322 "cmp rscratch2, rscratch1, ASR #63\n\t"
14323 "b$cmp $labl" %}
14324 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14325 ins_encode %{
14326 Label* L = $labl$$label;
14327 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14328 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14329 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14330 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14331 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14332 %}
14333
14334 ins_pipe(pipe_serial);
14335 %}
14336
14337 // ============================================================================
14338 // Compare Instructions
14339
14340 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14341 %{
14342 match(Set cr (CmpI op1 op2));
14343
14344 effect(DEF cr, USE op1, USE op2);
14345
14346 ins_cost(INSN_COST);
14347 format %{ "cmpw $op1, $op2" %}
14348
14349 ins_encode(aarch64_enc_cmpw(op1, op2));
14350
14351 ins_pipe(icmp_reg_reg);
14352 %}
14353
14354 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14355 %{
14356 match(Set cr (CmpI op1 zero));
14357
14358 effect(DEF cr, USE op1);
14359
14360 ins_cost(INSN_COST);
14361 format %{ "cmpw $op1, 0" %}
14362
14363 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14364
14365 ins_pipe(icmp_reg_imm);
14366 %}
14367
14368 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14369 %{
14370 match(Set cr (CmpI op1 op2));
14371
14372 effect(DEF cr, USE op1);
14373
14374 ins_cost(INSN_COST);
14375 format %{ "cmpw $op1, $op2" %}
14376
14377 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14378
14379 ins_pipe(icmp_reg_imm);
14380 %}
14381
14382 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14383 %{
14384 match(Set cr (CmpI op1 op2));
14385
14386 effect(DEF cr, USE op1);
14387
14388 ins_cost(INSN_COST * 2);
14389 format %{ "cmpw $op1, $op2" %}
14390
14391 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14392
14393 ins_pipe(icmp_reg_imm);
14394 %}
14395
14396 // Unsigned compare Instructions; really, same as signed compare
14397 // except it should only be used to feed an If or a CMovI which takes a
14398 // cmpOpU.
14399
14400 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14401 %{
14402 match(Set cr (CmpU op1 op2));
14403
14404 effect(DEF cr, USE op1, USE op2);
14405
14406 ins_cost(INSN_COST);
14407 format %{ "cmpw $op1, $op2\t# unsigned" %}
14408
14409 ins_encode(aarch64_enc_cmpw(op1, op2));
14410
14411 ins_pipe(icmp_reg_reg);
14412 %}
14413
14414 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14415 %{
14416 match(Set cr (CmpU op1 zero));
14417
14418 effect(DEF cr, USE op1);
14419
14420 ins_cost(INSN_COST);
14421 format %{ "cmpw $op1, #0\t# unsigned" %}
14422
14423 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14424
14425 ins_pipe(icmp_reg_imm);
14426 %}
14427
14428 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14429 %{
14430 match(Set cr (CmpU op1 op2));
14431
14432 effect(DEF cr, USE op1);
14433
14434 ins_cost(INSN_COST);
14435 format %{ "cmpw $op1, $op2\t# unsigned" %}
14436
14437 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14438
14439 ins_pipe(icmp_reg_imm);
14440 %}
14441
14442 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14443 %{
14444 match(Set cr (CmpU op1 op2));
14445
14446 effect(DEF cr, USE op1);
14447
14448 ins_cost(INSN_COST * 2);
14449 format %{ "cmpw $op1, $op2\t# unsigned" %}
14450
14451 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14452
14453 ins_pipe(icmp_reg_imm);
14454 %}
14455
14456 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14457 %{
14458 match(Set cr (CmpL op1 op2));
14459
14460 effect(DEF cr, USE op1, USE op2);
14461
14462 ins_cost(INSN_COST);
14463 format %{ "cmp $op1, $op2" %}
14464
14465 ins_encode(aarch64_enc_cmp(op1, op2));
14466
14467 ins_pipe(icmp_reg_reg);
14468 %}
14469
14470 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14471 %{
14472 match(Set cr (CmpL op1 zero));
14473
14474 effect(DEF cr, USE op1);
14475
14476 ins_cost(INSN_COST);
14477 format %{ "tst $op1" %}
14478
14479 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14480
14481 ins_pipe(icmp_reg_imm);
14482 %}
14483
14484 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14485 %{
14486 match(Set cr (CmpL op1 op2));
14487
14488 effect(DEF cr, USE op1);
14489
14490 ins_cost(INSN_COST);
14491 format %{ "cmp $op1, $op2" %}
14492
14493 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14494
14495 ins_pipe(icmp_reg_imm);
14496 %}
14497
14498 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14499 %{
14500 match(Set cr (CmpL op1 op2));
14501
14502 effect(DEF cr, USE op1);
14503
14504 ins_cost(INSN_COST * 2);
14505 format %{ "cmp $op1, $op2" %}
14506
14507 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14508
14509 ins_pipe(icmp_reg_imm);
14510 %}
14511
14512 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14513 %{
14514 match(Set cr (CmpUL op1 op2));
14515
14516 effect(DEF cr, USE op1, USE op2);
14517
14518 ins_cost(INSN_COST);
14519 format %{ "cmp $op1, $op2" %}
14520
14521 ins_encode(aarch64_enc_cmp(op1, op2));
14522
14523 ins_pipe(icmp_reg_reg);
14524 %}
14525
14526 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14527 %{
14528 match(Set cr (CmpUL op1 zero));
14529
14530 effect(DEF cr, USE op1);
14531
14532 ins_cost(INSN_COST);
14533 format %{ "tst $op1" %}
14534
14535 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14536
14537 ins_pipe(icmp_reg_imm);
14538 %}
14539
14540 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14541 %{
14542 match(Set cr (CmpUL op1 op2));
14543
14544 effect(DEF cr, USE op1);
14545
14546 ins_cost(INSN_COST);
14547 format %{ "cmp $op1, $op2" %}
14548
14549 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14550
14551 ins_pipe(icmp_reg_imm);
14552 %}
14553
14554 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14555 %{
14556 match(Set cr (CmpUL op1 op2));
14557
14558 effect(DEF cr, USE op1);
14559
14560 ins_cost(INSN_COST * 2);
14561 format %{ "cmp $op1, $op2" %}
14562
14563 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14564
14565 ins_pipe(icmp_reg_imm);
14566 %}
14567
14568 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14569 %{
14570 match(Set cr (CmpP op1 op2));
14571
14572 effect(DEF cr, USE op1, USE op2);
14573
14574 ins_cost(INSN_COST);
14575 format %{ "cmp $op1, $op2\t // ptr" %}
14576
14577 ins_encode(aarch64_enc_cmpp(op1, op2));
14578
14579 ins_pipe(icmp_reg_reg);
14580 %}
14581
14582 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14583 %{
14584 match(Set cr (CmpN op1 op2));
14585
14586 effect(DEF cr, USE op1, USE op2);
14587
14588 ins_cost(INSN_COST);
14589 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14590
14591 ins_encode(aarch64_enc_cmpn(op1, op2));
14592
14593 ins_pipe(icmp_reg_reg);
14594 %}
14595
14596 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14597 %{
14598 match(Set cr (CmpP op1 zero));
14599
14600 effect(DEF cr, USE op1, USE zero);
14601
14602 ins_cost(INSN_COST);
14603 format %{ "cmp $op1, 0\t // ptr" %}
14604
14605 ins_encode(aarch64_enc_testp(op1));
14606
14607 ins_pipe(icmp_reg_imm);
14608 %}
14609
14610 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14611 %{
14612 match(Set cr (CmpN op1 zero));
14613
14614 effect(DEF cr, USE op1, USE zero);
14615
14616 ins_cost(INSN_COST);
14617 format %{ "cmp $op1, 0\t // compressed ptr" %}
14618
14619 ins_encode(aarch64_enc_testn(op1));
14620
14621 ins_pipe(icmp_reg_imm);
14622 %}
14623
14624 // FP comparisons
14625 //
14626 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14627 // using normal cmpOp. See declaration of rFlagsReg for details.
14628
14629 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14630 %{
14631 match(Set cr (CmpF src1 src2));
14632
14633 ins_cost(3 * INSN_COST);
14634 format %{ "fcmps $src1, $src2" %}
14635
14636 ins_encode %{
14637 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14638 %}
14639
14640 ins_pipe(pipe_class_compare);
14641 %}
14642
14643 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14644 %{
14645 match(Set cr (CmpF src1 src2));
14646
14647 ins_cost(3 * INSN_COST);
14648 format %{ "fcmps $src1, 0.0" %}
14649
14650 ins_encode %{
14651 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14652 %}
14653
14654 ins_pipe(pipe_class_compare);
14655 %}
14656 // FROM HERE
14657
14658 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14659 %{
14660 match(Set cr (CmpD src1 src2));
14661
14662 ins_cost(3 * INSN_COST);
14663 format %{ "fcmpd $src1, $src2" %}
14664
14665 ins_encode %{
14666 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14667 %}
14668
14669 ins_pipe(pipe_class_compare);
14670 %}
14671
14672 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14673 %{
14674 match(Set cr (CmpD src1 src2));
14675
14676 ins_cost(3 * INSN_COST);
14677 format %{ "fcmpd $src1, 0.0" %}
14678
14679 ins_encode %{
14680 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14681 %}
14682
14683 ins_pipe(pipe_class_compare);
14684 %}
14685
14686 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14687 %{
14688 match(Set dst (CmpF3 src1 src2));
14689 effect(KILL cr);
14690
14691 ins_cost(5 * INSN_COST);
14692 format %{ "fcmps $src1, $src2\n\t"
14693 "csinvw($dst, zr, zr, eq\n\t"
14694 "csnegw($dst, $dst, $dst, lt)"
14695 %}
14696
14697 ins_encode %{
14698 Label done;
14699 FloatRegister s1 = as_FloatRegister($src1$$reg);
14700 FloatRegister s2 = as_FloatRegister($src2$$reg);
14701 Register d = as_Register($dst$$reg);
14702 __ fcmps(s1, s2);
14703 // installs 0 if EQ else -1
14704 __ csinvw(d, zr, zr, Assembler::EQ);
14705 // keeps -1 if less or unordered else installs 1
14706 __ csnegw(d, d, d, Assembler::LT);
14707 __ bind(done);
14708 %}
14709
14710 ins_pipe(pipe_class_default);
14711
14712 %}
14713
14714 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14715 %{
14716 match(Set dst (CmpD3 src1 src2));
14717 effect(KILL cr);
14718
14719 ins_cost(5 * INSN_COST);
14720 format %{ "fcmpd $src1, $src2\n\t"
14721 "csinvw($dst, zr, zr, eq\n\t"
14722 "csnegw($dst, $dst, $dst, lt)"
14723 %}
14724
14725 ins_encode %{
14726 Label done;
14727 FloatRegister s1 = as_FloatRegister($src1$$reg);
14728 FloatRegister s2 = as_FloatRegister($src2$$reg);
14729 Register d = as_Register($dst$$reg);
14730 __ fcmpd(s1, s2);
14731 // installs 0 if EQ else -1
14732 __ csinvw(d, zr, zr, Assembler::EQ);
14733 // keeps -1 if less or unordered else installs 1
14734 __ csnegw(d, d, d, Assembler::LT);
14735 __ bind(done);
14736 %}
14737 ins_pipe(pipe_class_default);
14738
14739 %}
14740
14741 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14742 %{
14743 match(Set dst (CmpF3 src1 zero));
14744 effect(KILL cr);
14745
14746 ins_cost(5 * INSN_COST);
14747 format %{ "fcmps $src1, 0.0\n\t"
14748 "csinvw($dst, zr, zr, eq\n\t"
14749 "csnegw($dst, $dst, $dst, lt)"
14750 %}
14751
14752 ins_encode %{
14753 Label done;
14754 FloatRegister s1 = as_FloatRegister($src1$$reg);
14755 Register d = as_Register($dst$$reg);
14756 __ fcmps(s1, 0.0);
14757 // installs 0 if EQ else -1
14758 __ csinvw(d, zr, zr, Assembler::EQ);
14759 // keeps -1 if less or unordered else installs 1
14760 __ csnegw(d, d, d, Assembler::LT);
14761 __ bind(done);
14762 %}
14763
14764 ins_pipe(pipe_class_default);
14765
14766 %}
14767
14768 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14769 %{
14770 match(Set dst (CmpD3 src1 zero));
14771 effect(KILL cr);
14772
14773 ins_cost(5 * INSN_COST);
14774 format %{ "fcmpd $src1, 0.0\n\t"
14775 "csinvw($dst, zr, zr, eq\n\t"
14776 "csnegw($dst, $dst, $dst, lt)"
14777 %}
14778
14779 ins_encode %{
14780 Label done;
14781 FloatRegister s1 = as_FloatRegister($src1$$reg);
14782 Register d = as_Register($dst$$reg);
14783 __ fcmpd(s1, 0.0);
14784 // installs 0 if EQ else -1
14785 __ csinvw(d, zr, zr, Assembler::EQ);
14786 // keeps -1 if less or unordered else installs 1
14787 __ csnegw(d, d, d, Assembler::LT);
14788 __ bind(done);
14789 %}
14790 ins_pipe(pipe_class_default);
14791
14792 %}
14793
14794 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14795 %{
14796 match(Set dst (CmpLTMask p q));
14797 effect(KILL cr);
14798
14799 ins_cost(3 * INSN_COST);
14800
14801 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14802 "csetw $dst, lt\n\t"
14803 "subw $dst, zr, $dst"
14804 %}
14805
14806 ins_encode %{
14807 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14808 __ csetw(as_Register($dst$$reg), Assembler::LT);
14809 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14810 %}
14811
14812 ins_pipe(ialu_reg_reg);
14813 %}
14814
14815 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14816 %{
14817 match(Set dst (CmpLTMask src zero));
14818 effect(KILL cr);
14819
14820 ins_cost(INSN_COST);
14821
14822 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14823
14824 ins_encode %{
14825 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14826 %}
14827
14828 ins_pipe(ialu_reg_shift);
14829 %}
14830
14831 // ============================================================================
14832 // Max and Min
14833
14834 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14835
14836 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14837 %{
14838 effect(DEF cr, USE src);
14839 ins_cost(INSN_COST);
14840 format %{ "cmpw $src, 0" %}
14841
14842 ins_encode %{
14843 __ cmpw($src$$Register, 0);
14844 %}
14845 ins_pipe(icmp_reg_imm);
14846 %}
14847
14848 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14849 %{
14850 match(Set dst (MinI src1 src2));
14851 ins_cost(INSN_COST * 3);
14852
14853 expand %{
14854 rFlagsReg cr;
14855 compI_reg_reg(cr, src1, src2);
14856 cmovI_reg_reg_lt(dst, src1, src2, cr);
14857 %}
14858 %}
14859
14860 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14861 %{
14862 match(Set dst (MaxI src1 src2));
14863 ins_cost(INSN_COST * 3);
14864
14865 expand %{
14866 rFlagsReg cr;
14867 compI_reg_reg(cr, src1, src2);
14868 cmovI_reg_reg_gt(dst, src1, src2, cr);
14869 %}
14870 %}
14871
14872
14873 // ============================================================================
14874 // Branch Instructions
14875
14876 // Direct Branch.
14877 instruct branch(label lbl)
14878 %{
14879 match(Goto);
14880
14881 effect(USE lbl);
14882
14883 ins_cost(BRANCH_COST);
14884 format %{ "b $lbl" %}
14885
14886 ins_encode(aarch64_enc_b(lbl));
14887
14888 ins_pipe(pipe_branch);
14889 %}
14890
14891 // Conditional Near Branch
14892 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14893 %{
14894 // Same match rule as `branchConFar'.
14895 match(If cmp cr);
14896
14897 effect(USE lbl);
14898
14899 ins_cost(BRANCH_COST);
14900 // If set to 1 this indicates that the current instruction is a
14901 // short variant of a long branch. This avoids using this
14902 // instruction in first-pass matching. It will then only be used in
14903 // the `Shorten_branches' pass.
14904 // ins_short_branch(1);
14905 format %{ "b$cmp $lbl" %}
14906
14907 ins_encode(aarch64_enc_br_con(cmp, lbl));
14908
14909 ins_pipe(pipe_branch_cond);
14910 %}
14911
14912 // Conditional Near Branch Unsigned
14913 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14914 %{
14915 // Same match rule as `branchConFar'.
14916 match(If cmp cr);
14917
14918 effect(USE lbl);
14919
14920 ins_cost(BRANCH_COST);
14921 // If set to 1 this indicates that the current instruction is a
14922 // short variant of a long branch. This avoids using this
14923 // instruction in first-pass matching. It will then only be used in
14924 // the `Shorten_branches' pass.
14925 // ins_short_branch(1);
14926 format %{ "b$cmp $lbl\t# unsigned" %}
14927
14928 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14929
14930 ins_pipe(pipe_branch_cond);
14931 %}
14932
14933 // Make use of CBZ and CBNZ. These instructions, as well as being
14934 // shorter than (cmp; branch), have the additional benefit of not
14935 // killing the flags.
14936
14937 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14938 match(If cmp (CmpI op1 op2));
14939 effect(USE labl);
14940
14941 ins_cost(BRANCH_COST);
14942 format %{ "cbw$cmp $op1, $labl" %}
14943 ins_encode %{
14944 Label* L = $labl$$label;
14945 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14946 if (cond == Assembler::EQ)
14947 __ cbzw($op1$$Register, *L);
14948 else
14949 __ cbnzw($op1$$Register, *L);
14950 %}
14951 ins_pipe(pipe_cmp_branch);
14952 %}
14953
14954 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14955 match(If cmp (CmpL op1 op2));
14956 effect(USE labl);
14957
14958 ins_cost(BRANCH_COST);
14959 format %{ "cb$cmp $op1, $labl" %}
14960 ins_encode %{
14961 Label* L = $labl$$label;
14962 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14963 if (cond == Assembler::EQ)
14964 __ cbz($op1$$Register, *L);
14965 else
14966 __ cbnz($op1$$Register, *L);
14967 %}
14968 ins_pipe(pipe_cmp_branch);
14969 %}
14970
14971 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14972 match(If cmp (CmpP op1 op2));
14973 effect(USE labl);
14974
14975 ins_cost(BRANCH_COST);
14976 format %{ "cb$cmp $op1, $labl" %}
14977 ins_encode %{
14978 Label* L = $labl$$label;
14979 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14980 if (cond == Assembler::EQ)
14981 __ cbz($op1$$Register, *L);
14982 else
14983 __ cbnz($op1$$Register, *L);
14984 %}
14985 ins_pipe(pipe_cmp_branch);
14986 %}
14987
14988 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14989 match(If cmp (CmpN op1 op2));
14990 effect(USE labl);
14991
14992 ins_cost(BRANCH_COST);
14993 format %{ "cbw$cmp $op1, $labl" %}
14994 ins_encode %{
14995 Label* L = $labl$$label;
14996 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14997 if (cond == Assembler::EQ)
14998 __ cbzw($op1$$Register, *L);
14999 else
15000 __ cbnzw($op1$$Register, *L);
15001 %}
15002 ins_pipe(pipe_cmp_branch);
15003 %}
15004
15005 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15006 match(If cmp (CmpP (DecodeN oop) zero));
15007 effect(USE labl);
15008
15009 ins_cost(BRANCH_COST);
15010 format %{ "cb$cmp $oop, $labl" %}
15011 ins_encode %{
15012 Label* L = $labl$$label;
15013 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15014 if (cond == Assembler::EQ)
15015 __ cbzw($oop$$Register, *L);
15016 else
15017 __ cbnzw($oop$$Register, *L);
15018 %}
15019 ins_pipe(pipe_cmp_branch);
15020 %}
15021
15022 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15023 match(If cmp (CmpU op1 op2));
15024 effect(USE labl);
15025
15026 ins_cost(BRANCH_COST);
15027 format %{ "cbw$cmp $op1, $labl" %}
15028 ins_encode %{
15029 Label* L = $labl$$label;
15030 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15031 if (cond == Assembler::EQ || cond == Assembler::LS) {
15032 __ cbzw($op1$$Register, *L);
15033 } else {
15034 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15035 __ cbnzw($op1$$Register, *L);
15036 }
15037 %}
15038 ins_pipe(pipe_cmp_branch);
15039 %}
15040
15041 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15042 match(If cmp (CmpUL op1 op2));
15043 effect(USE labl);
15044
15045 ins_cost(BRANCH_COST);
15046 format %{ "cb$cmp $op1, $labl" %}
15047 ins_encode %{
15048 Label* L = $labl$$label;
15049 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15050 if (cond == Assembler::EQ || cond == Assembler::LS) {
15051 __ cbz($op1$$Register, *L);
15052 } else {
15053 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15054 __ cbnz($op1$$Register, *L);
15055 }
15056 %}
15057 ins_pipe(pipe_cmp_branch);
15058 %}
15059
15060 // Test bit and Branch
15061
15062 // Patterns for short (< 32KiB) variants
15063 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15064 match(If cmp (CmpL op1 op2));
15065 effect(USE labl);
15066
15067 ins_cost(BRANCH_COST);
15068 format %{ "cb$cmp $op1, $labl # long" %}
15069 ins_encode %{
15070 Label* L = $labl$$label;
15071 Assembler::Condition cond =
15072 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15073 __ tbr(cond, $op1$$Register, 63, *L);
15074 %}
15075 ins_pipe(pipe_cmp_branch);
15076 ins_short_branch(1);
15077 %}
15078
15079 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15080 match(If cmp (CmpI op1 op2));
15081 effect(USE labl);
15082
15083 ins_cost(BRANCH_COST);
15084 format %{ "cb$cmp $op1, $labl # int" %}
15085 ins_encode %{
15086 Label* L = $labl$$label;
15087 Assembler::Condition cond =
15088 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15089 __ tbr(cond, $op1$$Register, 31, *L);
15090 %}
15091 ins_pipe(pipe_cmp_branch);
15092 ins_short_branch(1);
15093 %}
15094
15095 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15096 match(If cmp (CmpL (AndL op1 op2) op3));
15097 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15098 effect(USE labl);
15099
15100 ins_cost(BRANCH_COST);
15101 format %{ "tb$cmp $op1, $op2, $labl" %}
15102 ins_encode %{
15103 Label* L = $labl$$label;
15104 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15105 int bit = exact_log2_long($op2$$constant);
15106 __ tbr(cond, $op1$$Register, bit, *L);
15107 %}
15108 ins_pipe(pipe_cmp_branch);
15109 ins_short_branch(1);
15110 %}
15111
15112 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15113 match(If cmp (CmpI (AndI op1 op2) op3));
15114 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15115 effect(USE labl);
15116
15117 ins_cost(BRANCH_COST);
15118 format %{ "tb$cmp $op1, $op2, $labl" %}
15119 ins_encode %{
15120 Label* L = $labl$$label;
15121 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15122 int bit = exact_log2((juint)$op2$$constant);
15123 __ tbr(cond, $op1$$Register, bit, *L);
15124 %}
15125 ins_pipe(pipe_cmp_branch);
15126 ins_short_branch(1);
15127 %}
15128
15129 // And far variants
15130 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15131 match(If cmp (CmpL op1 op2));
15132 effect(USE labl);
15133
15134 ins_cost(BRANCH_COST);
15135 format %{ "cb$cmp $op1, $labl # long" %}
15136 ins_encode %{
15137 Label* L = $labl$$label;
15138 Assembler::Condition cond =
15139 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15140 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15141 %}
15142 ins_pipe(pipe_cmp_branch);
15143 %}
15144
15145 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15146 match(If cmp (CmpI op1 op2));
15147 effect(USE labl);
15148
15149 ins_cost(BRANCH_COST);
15150 format %{ "cb$cmp $op1, $labl # int" %}
15151 ins_encode %{
15152 Label* L = $labl$$label;
15153 Assembler::Condition cond =
15154 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15155 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15156 %}
15157 ins_pipe(pipe_cmp_branch);
15158 %}
15159
15160 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15161 match(If cmp (CmpL (AndL op1 op2) op3));
15162 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15163 effect(USE labl);
15164
15165 ins_cost(BRANCH_COST);
15166 format %{ "tb$cmp $op1, $op2, $labl" %}
15167 ins_encode %{
15168 Label* L = $labl$$label;
15169 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15170 int bit = exact_log2_long($op2$$constant);
15171 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15172 %}
15173 ins_pipe(pipe_cmp_branch);
15174 %}
15175
15176 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15177 match(If cmp (CmpI (AndI op1 op2) op3));
15178 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15179 effect(USE labl);
15180
15181 ins_cost(BRANCH_COST);
15182 format %{ "tb$cmp $op1, $op2, $labl" %}
15183 ins_encode %{
15184 Label* L = $labl$$label;
15185 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15186 int bit = exact_log2((juint)$op2$$constant);
15187 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15188 %}
15189 ins_pipe(pipe_cmp_branch);
15190 %}
15191
15192 // Test bits
15193
15194 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15195 match(Set cr (CmpL (AndL op1 op2) op3));
15196 predicate(Assembler::operand_valid_for_logical_immediate
15197 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15198
15199 ins_cost(INSN_COST);
15200 format %{ "tst $op1, $op2 # long" %}
15201 ins_encode %{
15202 __ tst($op1$$Register, $op2$$constant);
15203 %}
15204 ins_pipe(ialu_reg_reg);
15205 %}
15206
15207 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15208 match(Set cr (CmpI (AndI op1 op2) op3));
15209 predicate(Assembler::operand_valid_for_logical_immediate
15210 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15211
15212 ins_cost(INSN_COST);
15213 format %{ "tst $op1, $op2 # int" %}
15214 ins_encode %{
15215 __ tstw($op1$$Register, $op2$$constant);
15216 %}
15217 ins_pipe(ialu_reg_reg);
15218 %}
15219
15220 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15221 match(Set cr (CmpL (AndL op1 op2) op3));
15222
15223 ins_cost(INSN_COST);
15224 format %{ "tst $op1, $op2 # long" %}
15225 ins_encode %{
15226 __ tst($op1$$Register, $op2$$Register);
15227 %}
15228 ins_pipe(ialu_reg_reg);
15229 %}
15230
15231 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15232 match(Set cr (CmpI (AndI op1 op2) op3));
15233
15234 ins_cost(INSN_COST);
15235 format %{ "tstw $op1, $op2 # int" %}
15236 ins_encode %{
15237 __ tstw($op1$$Register, $op2$$Register);
15238 %}
15239 ins_pipe(ialu_reg_reg);
15240 %}
15241
15242
15243 // Conditional Far Branch
15244 // Conditional Far Branch Unsigned
15245 // TODO: fixme
15246
15247 // counted loop end branch near
15248 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15249 %{
15250 match(CountedLoopEnd cmp cr);
15251
15252 effect(USE lbl);
15253
15254 ins_cost(BRANCH_COST);
15255 // short variant.
15256 // ins_short_branch(1);
15257 format %{ "b$cmp $lbl \t// counted loop end" %}
15258
15259 ins_encode(aarch64_enc_br_con(cmp, lbl));
15260
15261 ins_pipe(pipe_branch);
15262 %}
15263
15264 // counted loop end branch far
15265 // TODO: fixme
15266
15267 // ============================================================================
15268 // inlined locking and unlocking
15269
15270 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15271 %{
15272 match(Set cr (FastLock object box));
15273 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15274
15275 ins_cost(5 * INSN_COST);
15276 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15277
15278 ins_encode %{
15279 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15280 %}
15281
15282 ins_pipe(pipe_serial);
15283 %}
15284
15285 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15286 %{
15287 match(Set cr (FastUnlock object box));
15288 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15289
15290 ins_cost(5 * INSN_COST);
15291 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15292
15293 ins_encode %{
15294 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15295 %}
15296
15297 ins_pipe(pipe_serial);
15298 %}
15299
15300 // ============================================================================
15301 // Safepoint Instructions
15302
15303 // TODO
15304 // provide a near and far version of this code
15305
15306 instruct safePoint(rFlagsReg cr, iRegP poll)
15307 %{
15308 match(SafePoint poll);
15309 effect(KILL cr);
15310
15311 format %{
15312 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15313 %}
15314 ins_encode %{
15315 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15316 %}
15317 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15318 %}
15319
15320
15321 // ============================================================================
15322 // Procedure Call/Return Instructions
15323
15324 // Call Java Static Instruction
15325
15326 instruct CallStaticJavaDirect(method meth)
15327 %{
15328 match(CallStaticJava);
15329
15330 effect(USE meth);
15331
15332 ins_cost(CALL_COST);
15333
15334 format %{ "call,static $meth \t// ==> " %}
15335
15336 ins_encode(aarch64_enc_java_static_call(meth),
15337 aarch64_enc_call_epilog);
15338
15339 ins_pipe(pipe_class_call);
15340 %}
15341
15342 // TO HERE
15343
15344 // Call Java Dynamic Instruction
15345 instruct CallDynamicJavaDirect(method meth)
15346 %{
15347 match(CallDynamicJava);
15348
15349 effect(USE meth);
15350
15351 ins_cost(CALL_COST);
15352
15353 format %{ "CALL,dynamic $meth \t// ==> " %}
15354
15355 ins_encode(aarch64_enc_java_dynamic_call(meth),
15356 aarch64_enc_call_epilog);
15357
15358 ins_pipe(pipe_class_call);
15359 %}
15360
15361 // Call Runtime Instruction
15362
15363 instruct CallRuntimeDirect(method meth)
15364 %{
15365 match(CallRuntime);
15366
15367 effect(USE meth);
15368
15369 ins_cost(CALL_COST);
15370
15371 format %{ "CALL, runtime $meth" %}
15372
15373 ins_encode( aarch64_enc_java_to_runtime(meth) );
15374
15375 ins_pipe(pipe_class_call);
15376 %}
15377
15378 // Call Runtime Instruction
15379
15380 instruct CallLeafDirect(method meth)
15381 %{
15382 match(CallLeaf);
15383
15384 effect(USE meth);
15385
15386 ins_cost(CALL_COST);
15387
15388 format %{ "CALL, runtime leaf $meth" %}
15389
15390 ins_encode( aarch64_enc_java_to_runtime(meth) );
15391
15392 ins_pipe(pipe_class_call);
15393 %}
15394
15395 // Call Runtime Instruction without safepoint and with vector arguments
15396 instruct CallLeafDirectVector(method meth)
15397 %{
15398 match(CallLeafVector);
15399
15400 effect(USE meth);
15401
15402 ins_cost(CALL_COST);
15403
15404 format %{ "CALL, runtime leaf vector $meth" %}
15405
15406 ins_encode(aarch64_enc_java_to_runtime(meth));
15407
15408 ins_pipe(pipe_class_call);
15409 %}
15410
15411 // Call Runtime Instruction
15412
15413 instruct CallLeafNoFPDirect(method meth)
15414 %{
15415 match(CallLeafNoFP);
15416
15417 effect(USE meth);
15418
15419 ins_cost(CALL_COST);
15420
15421 format %{ "CALL, runtime leaf nofp $meth" %}
15422
15423 ins_encode( aarch64_enc_java_to_runtime(meth) );
15424
15425 ins_pipe(pipe_class_call);
15426 %}
15427
15428 // Tail Call; Jump from runtime stub to Java code.
15429 // Also known as an 'interprocedural jump'.
15430 // Target of jump will eventually return to caller.
15431 // TailJump below removes the return address.
15432 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15433 // emitted just above the TailCall which has reset rfp to the caller state.
15434 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15435 %{
15436 match(TailCall jump_target method_ptr);
15437
15438 ins_cost(CALL_COST);
15439
15440 format %{ "br $jump_target\t# $method_ptr holds method" %}
15441
15442 ins_encode(aarch64_enc_tail_call(jump_target));
15443
15444 ins_pipe(pipe_class_call);
15445 %}
15446
15447 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15448 %{
15449 match(TailJump jump_target ex_oop);
15450
15451 ins_cost(CALL_COST);
15452
15453 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15454
15455 ins_encode(aarch64_enc_tail_jmp(jump_target));
15456
15457 ins_pipe(pipe_class_call);
15458 %}
15459
15460 // Forward exception.
15461 instruct ForwardExceptionjmp()
15462 %{
15463 match(ForwardException);
15464 ins_cost(CALL_COST);
15465
15466 format %{ "b forward_exception_stub" %}
15467 ins_encode %{
15468 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15469 %}
15470 ins_pipe(pipe_class_call);
15471 %}
15472
15473 // Create exception oop: created by stack-crawling runtime code.
15474 // Created exception is now available to this handler, and is setup
15475 // just prior to jumping to this handler. No code emitted.
15476 // TODO check
15477 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15478 instruct CreateException(iRegP_R0 ex_oop)
15479 %{
15480 match(Set ex_oop (CreateEx));
15481
15482 format %{ " -- \t// exception oop; no code emitted" %}
15483
15484 size(0);
15485
15486 ins_encode( /*empty*/ );
15487
15488 ins_pipe(pipe_class_empty);
15489 %}
15490
15491 // Rethrow exception: The exception oop will come in the first
15492 // argument position. Then JUMP (not call) to the rethrow stub code.
15493 instruct RethrowException() %{
15494 match(Rethrow);
15495 ins_cost(CALL_COST);
15496
15497 format %{ "b rethrow_stub" %}
15498
15499 ins_encode( aarch64_enc_rethrow() );
15500
15501 ins_pipe(pipe_class_call);
15502 %}
15503
15504
15505 // Return Instruction
15506 // epilog node loads ret address into lr as part of frame pop
15507 instruct Ret()
15508 %{
15509 match(Return);
15510
15511 format %{ "ret\t// return register" %}
15512
15513 ins_encode( aarch64_enc_ret() );
15514
15515 ins_pipe(pipe_branch);
15516 %}
15517
15518 // Die now.
15519 instruct ShouldNotReachHere() %{
15520 match(Halt);
15521
15522 ins_cost(CALL_COST);
15523 format %{ "ShouldNotReachHere" %}
15524
15525 ins_encode %{
15526 if (is_reachable()) {
15527 const char* str = __ code_string(_halt_reason);
15528 __ stop(str);
15529 }
15530 %}
15531
15532 ins_pipe(pipe_class_default);
15533 %}
15534
15535 // ============================================================================
15536 // Partial Subtype Check
15537 //
15538 // superklass array for an instance of the superklass. Set a hidden
15539 // internal cache on a hit (cache is checked with exposed code in
15540 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15541 // encoding ALSO sets flags.
15542
15543 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15544 %{
15545 match(Set result (PartialSubtypeCheck sub super));
15546 predicate(!UseSecondarySupersTable);
15547 effect(KILL cr, KILL temp);
15548
15549 ins_cost(20 * INSN_COST); // slightly larger than the next version
15550 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15551
15552 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15553
15554 opcode(0x1); // Force zero of result reg on hit
15555
15556 ins_pipe(pipe_class_memory);
15557 %}
15558
15559 // Two versions of partialSubtypeCheck, both used when we need to
15560 // search for a super class in the secondary supers array. The first
15561 // is used when we don't know _a priori_ the class being searched
15562 // for. The second, far more common, is used when we do know: this is
15563 // used for instanceof, checkcast, and any case where C2 can determine
15564 // it by constant propagation.
15565
15566 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15567 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15568 rFlagsReg cr)
15569 %{
15570 match(Set result (PartialSubtypeCheck sub super));
15571 predicate(UseSecondarySupersTable);
15572 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15573
15574 ins_cost(10 * INSN_COST); // slightly larger than the next version
15575 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15576
15577 ins_encode %{
15578 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15579 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15580 $vtemp$$FloatRegister,
15581 $result$$Register, /*L_success*/nullptr);
15582 %}
15583
15584 ins_pipe(pipe_class_memory);
15585 %}
15586
15587 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15588 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15589 rFlagsReg cr)
15590 %{
15591 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15592 predicate(UseSecondarySupersTable);
15593 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15594
15595 ins_cost(5 * INSN_COST); // smaller than the next version
15596 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15597
15598 ins_encode %{
15599 bool success = false;
15600 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15601 if (InlineSecondarySupersTest) {
15602 success =
15603 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15604 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15605 $vtemp$$FloatRegister,
15606 $result$$Register,
15607 super_klass_slot);
15608 } else {
15609 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15610 success = (call != nullptr);
15611 }
15612 if (!success) {
15613 ciEnv::current()->record_failure("CodeCache is full");
15614 return;
15615 }
15616 %}
15617
15618 ins_pipe(pipe_class_memory);
15619 %}
15620
15621 // Intrisics for String.compareTo()
15622
15623 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15624 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15625 %{
15626 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15627 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15628 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15629
15630 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15631 ins_encode %{
15632 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15633 __ string_compare($str1$$Register, $str2$$Register,
15634 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15635 $tmp1$$Register, $tmp2$$Register,
15636 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15637 %}
15638 ins_pipe(pipe_class_memory);
15639 %}
15640
15641 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15642 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15643 %{
15644 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15645 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15646 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15647
15648 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15649 ins_encode %{
15650 __ string_compare($str1$$Register, $str2$$Register,
15651 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15652 $tmp1$$Register, $tmp2$$Register,
15653 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15654 %}
15655 ins_pipe(pipe_class_memory);
15656 %}
15657
15658 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15659 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15660 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15661 %{
15662 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15663 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15664 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15665 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15666
15667 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15668 ins_encode %{
15669 __ string_compare($str1$$Register, $str2$$Register,
15670 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15671 $tmp1$$Register, $tmp2$$Register,
15672 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15673 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15674 %}
15675 ins_pipe(pipe_class_memory);
15676 %}
15677
15678 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15679 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15680 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15681 %{
15682 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15683 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15684 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15685 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15686
15687 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15688 ins_encode %{
15689 __ string_compare($str1$$Register, $str2$$Register,
15690 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15691 $tmp1$$Register, $tmp2$$Register,
15692 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15693 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15694 %}
15695 ins_pipe(pipe_class_memory);
15696 %}
15697
15698 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15699 // these string_compare variants as NEON register type for convenience so that the prototype of
15700 // string_compare can be shared with all variants.
15701
15702 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15703 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15704 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15705 pRegGov_P1 pgtmp2, rFlagsReg cr)
15706 %{
15707 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15708 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15709 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15710 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15711
15712 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15713 ins_encode %{
15714 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15715 __ string_compare($str1$$Register, $str2$$Register,
15716 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15717 $tmp1$$Register, $tmp2$$Register,
15718 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15719 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15720 StrIntrinsicNode::LL);
15721 %}
15722 ins_pipe(pipe_class_memory);
15723 %}
15724
15725 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15726 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15727 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15728 pRegGov_P1 pgtmp2, rFlagsReg cr)
15729 %{
15730 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15731 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15732 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15733 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15734
15735 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15736 ins_encode %{
15737 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15738 __ string_compare($str1$$Register, $str2$$Register,
15739 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15740 $tmp1$$Register, $tmp2$$Register,
15741 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15742 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15743 StrIntrinsicNode::LU);
15744 %}
15745 ins_pipe(pipe_class_memory);
15746 %}
15747
15748 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15749 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15750 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15751 pRegGov_P1 pgtmp2, rFlagsReg cr)
15752 %{
15753 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15754 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15755 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15756 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15757
15758 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15759 ins_encode %{
15760 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15761 __ string_compare($str1$$Register, $str2$$Register,
15762 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15763 $tmp1$$Register, $tmp2$$Register,
15764 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15765 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15766 StrIntrinsicNode::UL);
15767 %}
15768 ins_pipe(pipe_class_memory);
15769 %}
15770
15771 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15772 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15773 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15774 pRegGov_P1 pgtmp2, rFlagsReg cr)
15775 %{
15776 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15777 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15778 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15779 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15780
15781 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15782 ins_encode %{
15783 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15784 __ string_compare($str1$$Register, $str2$$Register,
15785 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15786 $tmp1$$Register, $tmp2$$Register,
15787 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15788 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15789 StrIntrinsicNode::UU);
15790 %}
15791 ins_pipe(pipe_class_memory);
15792 %}
15793
15794 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15795 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15796 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15797 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15798 %{
15799 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15800 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15801 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15802 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15803 TEMP vtmp0, TEMP vtmp1, KILL cr);
15804 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15805 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15806
15807 ins_encode %{
15808 __ string_indexof($str1$$Register, $str2$$Register,
15809 $cnt1$$Register, $cnt2$$Register,
15810 $tmp1$$Register, $tmp2$$Register,
15811 $tmp3$$Register, $tmp4$$Register,
15812 $tmp5$$Register, $tmp6$$Register,
15813 -1, $result$$Register, StrIntrinsicNode::UU);
15814 %}
15815 ins_pipe(pipe_class_memory);
15816 %}
15817
15818 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15819 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15820 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15821 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15822 %{
15823 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15824 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15825 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15826 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15827 TEMP vtmp0, TEMP vtmp1, KILL cr);
15828 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15829 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15830
15831 ins_encode %{
15832 __ string_indexof($str1$$Register, $str2$$Register,
15833 $cnt1$$Register, $cnt2$$Register,
15834 $tmp1$$Register, $tmp2$$Register,
15835 $tmp3$$Register, $tmp4$$Register,
15836 $tmp5$$Register, $tmp6$$Register,
15837 -1, $result$$Register, StrIntrinsicNode::LL);
15838 %}
15839 ins_pipe(pipe_class_memory);
15840 %}
15841
15842 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15843 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15844 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15845 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15846 %{
15847 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15848 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15849 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15850 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15851 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15852 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15853 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15854
15855 ins_encode %{
15856 __ string_indexof($str1$$Register, $str2$$Register,
15857 $cnt1$$Register, $cnt2$$Register,
15858 $tmp1$$Register, $tmp2$$Register,
15859 $tmp3$$Register, $tmp4$$Register,
15860 $tmp5$$Register, $tmp6$$Register,
15861 -1, $result$$Register, StrIntrinsicNode::UL);
15862 %}
15863 ins_pipe(pipe_class_memory);
15864 %}
15865
15866 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15867 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15868 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15869 %{
15870 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15871 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15872 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15873 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15874 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15875 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15876
15877 ins_encode %{
15878 int icnt2 = (int)$int_cnt2$$constant;
15879 __ string_indexof($str1$$Register, $str2$$Register,
15880 $cnt1$$Register, zr,
15881 $tmp1$$Register, $tmp2$$Register,
15882 $tmp3$$Register, $tmp4$$Register, zr, zr,
15883 icnt2, $result$$Register, StrIntrinsicNode::UU);
15884 %}
15885 ins_pipe(pipe_class_memory);
15886 %}
15887
15888 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15889 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15890 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15891 %{
15892 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15893 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15894 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15895 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15896 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15897 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15898
15899 ins_encode %{
15900 int icnt2 = (int)$int_cnt2$$constant;
15901 __ string_indexof($str1$$Register, $str2$$Register,
15902 $cnt1$$Register, zr,
15903 $tmp1$$Register, $tmp2$$Register,
15904 $tmp3$$Register, $tmp4$$Register, zr, zr,
15905 icnt2, $result$$Register, StrIntrinsicNode::LL);
15906 %}
15907 ins_pipe(pipe_class_memory);
15908 %}
15909
15910 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15911 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15912 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15913 %{
15914 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15915 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15916 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15917 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15918 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15919 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15920
15921 ins_encode %{
15922 int icnt2 = (int)$int_cnt2$$constant;
15923 __ string_indexof($str1$$Register, $str2$$Register,
15924 $cnt1$$Register, zr,
15925 $tmp1$$Register, $tmp2$$Register,
15926 $tmp3$$Register, $tmp4$$Register, zr, zr,
15927 icnt2, $result$$Register, StrIntrinsicNode::UL);
15928 %}
15929 ins_pipe(pipe_class_memory);
15930 %}
15931
15932 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15933 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15934 iRegINoSp tmp3, rFlagsReg cr)
15935 %{
15936 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15937 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15938 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15939 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15940
15941 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15942
15943 ins_encode %{
15944 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15945 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15946 $tmp3$$Register);
15947 %}
15948 ins_pipe(pipe_class_memory);
15949 %}
15950
15951 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15952 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15953 iRegINoSp tmp3, rFlagsReg cr)
15954 %{
15955 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15956 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15957 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15958 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15959
15960 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15961
15962 ins_encode %{
15963 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15964 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15965 $tmp3$$Register);
15966 %}
15967 ins_pipe(pipe_class_memory);
15968 %}
15969
15970 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15971 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15972 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15973 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15974 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15975 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15976 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15977 ins_encode %{
15978 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15979 $result$$Register, $ztmp1$$FloatRegister,
15980 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15981 $ptmp$$PRegister, true /* isL */);
15982 %}
15983 ins_pipe(pipe_class_memory);
15984 %}
15985
15986 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15987 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15988 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15989 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15990 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15991 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15992 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15993 ins_encode %{
15994 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15995 $result$$Register, $ztmp1$$FloatRegister,
15996 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15997 $ptmp$$PRegister, false /* isL */);
15998 %}
15999 ins_pipe(pipe_class_memory);
16000 %}
16001
16002 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16003 iRegI_R0 result, rFlagsReg cr)
16004 %{
16005 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16006 match(Set result (StrEquals (Binary str1 str2) cnt));
16007 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16008
16009 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16010 ins_encode %{
16011 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16012 __ string_equals($str1$$Register, $str2$$Register,
16013 $result$$Register, $cnt$$Register);
16014 %}
16015 ins_pipe(pipe_class_memory);
16016 %}
16017
16018 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16019 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16020 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16021 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16022 iRegP_R10 tmp, rFlagsReg cr)
16023 %{
16024 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16025 match(Set result (AryEq ary1 ary2));
16026 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16027 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16028 TEMP vtmp6, TEMP vtmp7, KILL cr);
16029
16030 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16031 ins_encode %{
16032 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16033 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16034 $result$$Register, $tmp$$Register, 1);
16035 if (tpc == nullptr) {
16036 ciEnv::current()->record_failure("CodeCache is full");
16037 return;
16038 }
16039 %}
16040 ins_pipe(pipe_class_memory);
16041 %}
16042
16043 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16044 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16045 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16046 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16047 iRegP_R10 tmp, rFlagsReg cr)
16048 %{
16049 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16050 match(Set result (AryEq ary1 ary2));
16051 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16052 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16053 TEMP vtmp6, TEMP vtmp7, KILL cr);
16054
16055 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16056 ins_encode %{
16057 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16058 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16059 $result$$Register, $tmp$$Register, 2);
16060 if (tpc == nullptr) {
16061 ciEnv::current()->record_failure("CodeCache is full");
16062 return;
16063 }
16064 %}
16065 ins_pipe(pipe_class_memory);
16066 %}
16067
16068 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16069 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16070 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16071 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16072 %{
16073 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16074 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16075 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16076
16077 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16078 ins_encode %{
16079 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16080 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16081 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16082 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16083 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16084 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16085 (BasicType)$basic_type$$constant);
16086 if (tpc == nullptr) {
16087 ciEnv::current()->record_failure("CodeCache is full");
16088 return;
16089 }
16090 %}
16091 ins_pipe(pipe_class_memory);
16092 %}
16093
16094 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16095 %{
16096 match(Set result (CountPositives ary1 len));
16097 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16098 format %{ "count positives byte[] $ary1,$len -> $result" %}
16099 ins_encode %{
16100 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16101 if (tpc == nullptr) {
16102 ciEnv::current()->record_failure("CodeCache is full");
16103 return;
16104 }
16105 %}
16106 ins_pipe( pipe_slow );
16107 %}
16108
16109 // fast char[] to byte[] compression
16110 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16111 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16112 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16113 iRegI_R0 result, rFlagsReg cr)
16114 %{
16115 match(Set result (StrCompressedCopy src (Binary dst len)));
16116 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16117 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16118
16119 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16120 ins_encode %{
16121 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16122 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16123 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16124 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16125 %}
16126 ins_pipe(pipe_slow);
16127 %}
16128
16129 // fast byte[] to char[] inflation
16130 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16131 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16132 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16133 %{
16134 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16135 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16136 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16137 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16138
16139 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16140 ins_encode %{
16141 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16142 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16143 $vtmp2$$FloatRegister, $tmp$$Register);
16144 if (tpc == nullptr) {
16145 ciEnv::current()->record_failure("CodeCache is full");
16146 return;
16147 }
16148 %}
16149 ins_pipe(pipe_class_memory);
16150 %}
16151
16152 // encode char[] to byte[] in ISO_8859_1
16153 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16154 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16155 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16156 iRegI_R0 result, rFlagsReg cr)
16157 %{
16158 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16159 match(Set result (EncodeISOArray src (Binary dst len)));
16160 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16161 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16162
16163 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16164 ins_encode %{
16165 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16166 $result$$Register, false,
16167 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16168 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16169 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16170 %}
16171 ins_pipe(pipe_class_memory);
16172 %}
16173
16174 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16175 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16176 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16177 iRegI_R0 result, rFlagsReg cr)
16178 %{
16179 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16180 match(Set result (EncodeISOArray src (Binary dst len)));
16181 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16182 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16183
16184 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16185 ins_encode %{
16186 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16187 $result$$Register, true,
16188 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16189 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16190 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16191 %}
16192 ins_pipe(pipe_class_memory);
16193 %}
16194
16195 //----------------------------- CompressBits/ExpandBits ------------------------
16196
16197 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16198 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16199 match(Set dst (CompressBits src mask));
16200 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16201 format %{ "mov $tsrc, $src\n\t"
16202 "mov $tmask, $mask\n\t"
16203 "bext $tdst, $tsrc, $tmask\n\t"
16204 "mov $dst, $tdst"
16205 %}
16206 ins_encode %{
16207 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16208 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16209 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16210 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16211 %}
16212 ins_pipe(pipe_slow);
16213 %}
16214
16215 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16216 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16217 match(Set dst (CompressBits (LoadI mem) mask));
16218 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16219 format %{ "ldrs $tsrc, $mem\n\t"
16220 "ldrs $tmask, $mask\n\t"
16221 "bext $tdst, $tsrc, $tmask\n\t"
16222 "mov $dst, $tdst"
16223 %}
16224 ins_encode %{
16225 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16226 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16227 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16228 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16229 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16230 %}
16231 ins_pipe(pipe_slow);
16232 %}
16233
16234 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16235 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16236 match(Set dst (CompressBits src mask));
16237 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16238 format %{ "mov $tsrc, $src\n\t"
16239 "mov $tmask, $mask\n\t"
16240 "bext $tdst, $tsrc, $tmask\n\t"
16241 "mov $dst, $tdst"
16242 %}
16243 ins_encode %{
16244 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16245 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16246 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16247 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16248 %}
16249 ins_pipe(pipe_slow);
16250 %}
16251
16252 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16253 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16254 match(Set dst (CompressBits (LoadL mem) mask));
16255 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16256 format %{ "ldrd $tsrc, $mem\n\t"
16257 "ldrd $tmask, $mask\n\t"
16258 "bext $tdst, $tsrc, $tmask\n\t"
16259 "mov $dst, $tdst"
16260 %}
16261 ins_encode %{
16262 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16263 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16264 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16265 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16266 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16267 %}
16268 ins_pipe(pipe_slow);
16269 %}
16270
16271 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16272 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16273 match(Set dst (ExpandBits src mask));
16274 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16275 format %{ "mov $tsrc, $src\n\t"
16276 "mov $tmask, $mask\n\t"
16277 "bdep $tdst, $tsrc, $tmask\n\t"
16278 "mov $dst, $tdst"
16279 %}
16280 ins_encode %{
16281 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16282 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16283 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16284 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16285 %}
16286 ins_pipe(pipe_slow);
16287 %}
16288
16289 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16290 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16291 match(Set dst (ExpandBits (LoadI mem) mask));
16292 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16293 format %{ "ldrs $tsrc, $mem\n\t"
16294 "ldrs $tmask, $mask\n\t"
16295 "bdep $tdst, $tsrc, $tmask\n\t"
16296 "mov $dst, $tdst"
16297 %}
16298 ins_encode %{
16299 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16300 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16301 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16302 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16303 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16304 %}
16305 ins_pipe(pipe_slow);
16306 %}
16307
16308 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16309 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16310 match(Set dst (ExpandBits src mask));
16311 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16312 format %{ "mov $tsrc, $src\n\t"
16313 "mov $tmask, $mask\n\t"
16314 "bdep $tdst, $tsrc, $tmask\n\t"
16315 "mov $dst, $tdst"
16316 %}
16317 ins_encode %{
16318 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16319 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16320 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16321 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16322 %}
16323 ins_pipe(pipe_slow);
16324 %}
16325
16326
16327 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16328 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16329 match(Set dst (ExpandBits (LoadL mem) mask));
16330 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16331 format %{ "ldrd $tsrc, $mem\n\t"
16332 "ldrd $tmask, $mask\n\t"
16333 "bdep $tdst, $tsrc, $tmask\n\t"
16334 "mov $dst, $tdst"
16335 %}
16336 ins_encode %{
16337 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16338 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16339 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16340 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16341 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16342 %}
16343 ins_pipe(pipe_slow);
16344 %}
16345
16346 //----------------------------- Reinterpret ----------------------------------
16347 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16348 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16349 match(Set dst (ReinterpretHF2S src));
16350 format %{ "reinterpretHF2S $dst, $src" %}
16351 ins_encode %{
16352 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16353 %}
16354 ins_pipe(pipe_slow);
16355 %}
16356
16357 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16358 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16359 match(Set dst (ReinterpretS2HF src));
16360 format %{ "reinterpretS2HF $dst, $src" %}
16361 ins_encode %{
16362 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16363 %}
16364 ins_pipe(pipe_slow);
16365 %}
16366
16367 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16368 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16369 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16370 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16371 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16372 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16373 // can be omitted in this pattern, resulting in -
16374 // fcvt $dst, $src // Convert float to half-precision float
16375 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16376 %{
16377 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16378 format %{ "convF2HFAndS2HF $dst, $src" %}
16379 ins_encode %{
16380 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16381 %}
16382 ins_pipe(pipe_slow);
16383 %}
16384
16385 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16386 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16387 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16388 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16389 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16390 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16391 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16392 // resulting in -
16393 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16394 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16395 %{
16396 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16397 format %{ "convHF2SAndHF2F $dst, $src" %}
16398 ins_encode %{
16399 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16400 %}
16401 ins_pipe(pipe_slow);
16402 %}
16403
16404 // ============================================================================
16405 // This name is KNOWN by the ADLC and cannot be changed.
16406 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16407 // for this guy.
16408 instruct tlsLoadP(thread_RegP dst)
16409 %{
16410 match(Set dst (ThreadLocal));
16411
16412 ins_cost(0);
16413
16414 format %{ " -- \t// $dst=Thread::current(), empty" %}
16415
16416 size(0);
16417
16418 ins_encode( /*empty*/ );
16419
16420 ins_pipe(pipe_class_empty);
16421 %}
16422
16423 //----------PEEPHOLE RULES-----------------------------------------------------
16424 // These must follow all instruction definitions as they use the names
16425 // defined in the instructions definitions.
16426 //
16427 // peepmatch ( root_instr_name [preceding_instruction]* );
16428 //
16429 // peepconstraint %{
16430 // (instruction_number.operand_name relational_op instruction_number.operand_name
16431 // [, ...] );
16432 // // instruction numbers are zero-based using left to right order in peepmatch
16433 //
16434 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16435 // // provide an instruction_number.operand_name for each operand that appears
16436 // // in the replacement instruction's match rule
16437 //
16438 // ---------VM FLAGS---------------------------------------------------------
16439 //
16440 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16441 //
16442 // Each peephole rule is given an identifying number starting with zero and
16443 // increasing by one in the order seen by the parser. An individual peephole
16444 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16445 // on the command-line.
16446 //
16447 // ---------CURRENT LIMITATIONS----------------------------------------------
16448 //
16449 // Only match adjacent instructions in same basic block
16450 // Only equality constraints
16451 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16452 // Only one replacement instruction
16453 //
16454 // ---------EXAMPLE----------------------------------------------------------
16455 //
16456 // // pertinent parts of existing instructions in architecture description
16457 // instruct movI(iRegINoSp dst, iRegI src)
16458 // %{
16459 // match(Set dst (CopyI src));
16460 // %}
16461 //
16462 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16463 // %{
16464 // match(Set dst (AddI dst src));
16465 // effect(KILL cr);
16466 // %}
16467 //
16468 // // Change (inc mov) to lea
16469 // peephole %{
16470 // // increment preceded by register-register move
16471 // peepmatch ( incI_iReg movI );
16472 // // require that the destination register of the increment
16473 // // match the destination register of the move
16474 // peepconstraint ( 0.dst == 1.dst );
16475 // // construct a replacement instruction that sets
16476 // // the destination to ( move's source register + one )
16477 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16478 // %}
16479 //
16480
16481 // Implementation no longer uses movX instructions since
16482 // machine-independent system no longer uses CopyX nodes.
16483 //
16484 // peephole
16485 // %{
16486 // peepmatch (incI_iReg movI);
16487 // peepconstraint (0.dst == 1.dst);
16488 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16489 // %}
16490
16491 // peephole
16492 // %{
16493 // peepmatch (decI_iReg movI);
16494 // peepconstraint (0.dst == 1.dst);
16495 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16496 // %}
16497
16498 // peephole
16499 // %{
16500 // peepmatch (addI_iReg_imm movI);
16501 // peepconstraint (0.dst == 1.dst);
16502 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16503 // %}
16504
16505 // peephole
16506 // %{
16507 // peepmatch (incL_iReg movL);
16508 // peepconstraint (0.dst == 1.dst);
16509 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16510 // %}
16511
16512 // peephole
16513 // %{
16514 // peepmatch (decL_iReg movL);
16515 // peepconstraint (0.dst == 1.dst);
16516 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16517 // %}
16518
16519 // peephole
16520 // %{
16521 // peepmatch (addL_iReg_imm movL);
16522 // peepconstraint (0.dst == 1.dst);
16523 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16524 // %}
16525
16526 // peephole
16527 // %{
16528 // peepmatch (addP_iReg_imm movP);
16529 // peepconstraint (0.dst == 1.dst);
16530 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16531 // %}
16532
16533 // // Change load of spilled value to only a spill
16534 // instruct storeI(memory mem, iRegI src)
16535 // %{
16536 // match(Set mem (StoreI mem src));
16537 // %}
16538 //
16539 // instruct loadI(iRegINoSp dst, memory mem)
16540 // %{
16541 // match(Set dst (LoadI mem));
16542 // %}
16543 //
16544
16545 //----------SMARTSPILL RULES---------------------------------------------------
16546 // These must follow all instruction definitions as they use the names
16547 // defined in the instructions definitions.
16548
16549 // Local Variables:
16550 // mode: c++
16551 // End: