1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else {
1688 return 6 * NativeInstruction::instruction_size;
1689 }
1690 }
1691
1692 //=============================================================================
1693
1694 #ifndef PRODUCT
1695 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1696 st->print("BREAKPOINT");
1697 }
1698 #endif
1699
1700 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1701 __ brk(0);
1702 }
1703
1704 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1705 return MachNode::size(ra_);
1706 }
1707
1708 //=============================================================================
1709
1710 #ifndef PRODUCT
1711 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1712 st->print("nop \t# %d bytes pad for loops and calls", _count);
1713 }
1714 #endif
1715
1716 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1717 for (int i = 0; i < _count; i++) {
1718 __ nop();
1719 }
1720 }
1721
1722 uint MachNopNode::size(PhaseRegAlloc*) const {
1723 return _count * NativeInstruction::instruction_size;
1724 }
1725
1726 //=============================================================================
1727 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1728
1729 int ConstantTable::calculate_table_base_offset() const {
1730 return 0; // absolute addressing, no offset
1731 }
1732
1733 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1734 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1735 ShouldNotReachHere();
1736 }
1737
1738 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1739 // Empty encoding
1740 }
1741
1742 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1743 return 0;
1744 }
1745
1746 #ifndef PRODUCT
1747 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1748 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1749 }
1750 #endif
1751
1752 #ifndef PRODUCT
1753 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1754 Compile* C = ra_->C;
1755
1756 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1757
1758 if (C->output()->need_stack_bang(framesize))
1759 st->print("# stack bang size=%d\n\t", framesize);
1760
1761 if (VM_Version::use_rop_protection()) {
1762 st->print("ldr zr, [lr]\n\t");
1763 st->print("paciaz\n\t");
1764 }
1765 if (framesize < ((1 << 9) + 2 * wordSize)) {
1766 st->print("sub sp, sp, #%d\n\t", framesize);
1767 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1768 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1769 } else {
1770 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1771 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1772 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1773 st->print("sub sp, sp, rscratch1");
1774 }
1775 if (C->stub_function() == nullptr) {
1776 st->print("\n\t");
1777 st->print("ldr rscratch1, [guard]\n\t");
1778 st->print("dmb ishld\n\t");
1779 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1780 st->print("cmp rscratch1, rscratch2\n\t");
1781 st->print("b.eq skip");
1782 st->print("\n\t");
1783 st->print("blr #nmethod_entry_barrier_stub\n\t");
1784 st->print("b skip\n\t");
1785 st->print("guard: int\n\t");
1786 st->print("\n\t");
1787 st->print("skip:\n\t");
1788 }
1789 }
1790 #endif
1791
1792 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1793 Compile* C = ra_->C;
1794
1795 // n.b. frame size includes space for return pc and rfp
1796 const int framesize = C->output()->frame_size_in_bytes();
1797
1798 if (C->clinit_barrier_on_entry()) {
1799 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1800
1801 Label L_skip_barrier;
1802
1803 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1804 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1805 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1806 __ bind(L_skip_barrier);
1807 }
1808
1809 if (C->max_vector_size() > 0) {
1810 __ reinitialize_ptrue();
1811 }
1812
1813 int bangsize = C->output()->bang_size_in_bytes();
1814 if (C->output()->need_stack_bang(bangsize))
1815 __ generate_stack_overflow_check(bangsize);
1816
1817 __ build_frame(framesize);
1818
1819 if (C->stub_function() == nullptr) {
1820 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1821 // Dummy labels for just measuring the code size
1822 Label dummy_slow_path;
1823 Label dummy_continuation;
1824 Label dummy_guard;
1825 Label* slow_path = &dummy_slow_path;
1826 Label* continuation = &dummy_continuation;
1827 Label* guard = &dummy_guard;
1828 if (!Compile::current()->output()->in_scratch_emit_size()) {
1829 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1830 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1831 Compile::current()->output()->add_stub(stub);
1832 slow_path = &stub->entry();
1833 continuation = &stub->continuation();
1834 guard = &stub->guard();
1835 }
1836 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1837 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1838 }
1839
1840 if (VerifyStackAtCalls) {
1841 Unimplemented();
1842 }
1843
1844 C->output()->set_frame_complete(__ offset());
1845
1846 if (C->has_mach_constant_base_node()) {
1847 // NOTE: We set the table base offset here because users might be
1848 // emitted before MachConstantBaseNode.
1849 ConstantTable& constant_table = C->output()->constant_table();
1850 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1851 }
1852 }
1853
1854 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1855 {
1856 return MachNode::size(ra_); // too many variables; just compute it
1857 // the hard way
1858 }
1859
1860 int MachPrologNode::reloc() const
1861 {
1862 return 0;
1863 }
1864
1865 //=============================================================================
1866
1867 #ifndef PRODUCT
1868 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1869 Compile* C = ra_->C;
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 st->print("# pop frame %d\n\t",framesize);
1873
1874 if (framesize == 0) {
1875 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1876 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1877 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1878 st->print("add sp, sp, #%d\n\t", framesize);
1879 } else {
1880 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1881 st->print("add sp, sp, rscratch1\n\t");
1882 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1883 }
1884 if (VM_Version::use_rop_protection()) {
1885 st->print("autiaz\n\t");
1886 st->print("ldr zr, [lr]\n\t");
1887 }
1888
1889 if (do_polling() && C->is_method_compilation()) {
1890 st->print("# test polling word\n\t");
1891 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1892 st->print("cmp sp, rscratch1\n\t");
1893 st->print("bhi #slow_path");
1894 }
1895 }
1896 #endif
1897
1898 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1899 Compile* C = ra_->C;
1900 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1901
1902 __ remove_frame(framesize);
1903
1904 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1905 __ reserved_stack_check();
1906 }
1907
1908 if (do_polling() && C->is_method_compilation()) {
1909 Label dummy_label;
1910 Label* code_stub = &dummy_label;
1911 if (!C->output()->in_scratch_emit_size()) {
1912 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1913 C->output()->add_stub(stub);
1914 code_stub = &stub->entry();
1915 }
1916 __ relocate(relocInfo::poll_return_type);
1917 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1918 }
1919 }
1920
1921 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1922 // Variable size. Determine dynamically.
1923 return MachNode::size(ra_);
1924 }
1925
1926 int MachEpilogNode::reloc() const {
1927 // Return number of relocatable values contained in this instruction.
1928 return 1; // 1 for polling page.
1929 }
1930
1931 const Pipeline * MachEpilogNode::pipeline() const {
1932 return MachNode::pipeline_class();
1933 }
1934
1935 //=============================================================================
1936
1937 static enum RC rc_class(OptoReg::Name reg) {
1938
1939 if (reg == OptoReg::Bad) {
1940 return rc_bad;
1941 }
1942
1943 // we have 32 int registers * 2 halves
1944 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1945
1946 if (reg < slots_of_int_registers) {
1947 return rc_int;
1948 }
1949
1950 // we have 32 float register * 8 halves
1951 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1952 if (reg < slots_of_int_registers + slots_of_float_registers) {
1953 return rc_float;
1954 }
1955
1956 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1957 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1958 return rc_predicate;
1959 }
1960
1961 // Between predicate regs & stack is the flags.
1962 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1963
1964 return rc_stack;
1965 }
1966
1967 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1968 Compile* C = ra_->C;
1969
1970 // Get registers to move.
1971 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1972 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1973 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1974 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1975
1976 enum RC src_hi_rc = rc_class(src_hi);
1977 enum RC src_lo_rc = rc_class(src_lo);
1978 enum RC dst_hi_rc = rc_class(dst_hi);
1979 enum RC dst_lo_rc = rc_class(dst_lo);
1980
1981 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1982
1983 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1984 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1985 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1986 "expected aligned-adjacent pairs");
1987 }
1988
1989 if (src_lo == dst_lo && src_hi == dst_hi) {
1990 return 0; // Self copy, no move.
1991 }
1992
1993 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1994 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1995 int src_offset = ra_->reg2offset(src_lo);
1996 int dst_offset = ra_->reg2offset(dst_lo);
1997
1998 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1999 uint ireg = ideal_reg();
2000 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2001 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2002 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2003 if (ireg == Op_VecA && masm) {
2004 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2005 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2006 // stack->stack
2007 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2008 sve_vector_reg_size_in_bytes);
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2013 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2016 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]),
2018 as_FloatRegister(Matcher::_regEncode[src_lo]));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 } else if (masm) {
2023 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2024 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2025 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2026 // stack->stack
2027 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2028 if (ireg == Op_VecD) {
2029 __ unspill(rscratch1, true, src_offset);
2030 __ spill(rscratch1, true, dst_offset);
2031 } else {
2032 __ spill_copy128(src_offset, dst_offset);
2033 }
2034 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2035 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2036 ireg == Op_VecD ? __ T8B : __ T16B,
2037 as_FloatRegister(Matcher::_regEncode[src_lo]));
2038 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2039 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2040 ireg == Op_VecD ? __ D : __ Q,
2041 ra_->reg2offset(dst_lo));
2042 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2043 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 ireg == Op_VecD ? __ D : __ Q,
2045 ra_->reg2offset(src_lo));
2046 } else {
2047 ShouldNotReachHere();
2048 }
2049 }
2050 } else if (masm) {
2051 switch (src_lo_rc) {
2052 case rc_int:
2053 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2054 if (is64) {
2055 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_Register(Matcher::_regEncode[src_lo]));
2057 } else {
2058 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 }
2061 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2062 if (is64) {
2063 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_Register(Matcher::_regEncode[src_lo]));
2065 } else {
2066 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 }
2069 } else { // gpr --> stack spill
2070 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2071 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2072 }
2073 break;
2074 case rc_float:
2075 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2076 if (is64) {
2077 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2078 as_FloatRegister(Matcher::_regEncode[src_lo]));
2079 } else {
2080 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 }
2083 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2084 if (is64) {
2085 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2086 as_FloatRegister(Matcher::_regEncode[src_lo]));
2087 } else {
2088 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 }
2091 } else { // fpr --> stack spill
2092 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2093 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2094 is64 ? __ D : __ S, dst_offset);
2095 }
2096 break;
2097 case rc_stack:
2098 if (dst_lo_rc == rc_int) { // stack --> gpr load
2099 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2100 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2101 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2102 is64 ? __ D : __ S, src_offset);
2103 } else if (dst_lo_rc == rc_predicate) {
2104 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2105 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2106 } else { // stack --> stack copy
2107 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2108 if (ideal_reg() == Op_RegVectMask) {
2109 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2110 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2111 } else {
2112 __ unspill(rscratch1, is64, src_offset);
2113 __ spill(rscratch1, is64, dst_offset);
2114 }
2115 }
2116 break;
2117 case rc_predicate:
2118 if (dst_lo_rc == rc_predicate) {
2119 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2120 } else if (dst_lo_rc == rc_stack) {
2121 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2122 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2123 } else {
2124 assert(false, "bad src and dst rc_class combination.");
2125 ShouldNotReachHere();
2126 }
2127 break;
2128 default:
2129 assert(false, "bad rc_class for spill");
2130 ShouldNotReachHere();
2131 }
2132 }
2133
2134 if (st) {
2135 st->print("spill ");
2136 if (src_lo_rc == rc_stack) {
2137 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2138 } else {
2139 st->print("%s -> ", Matcher::regName[src_lo]);
2140 }
2141 if (dst_lo_rc == rc_stack) {
2142 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2143 } else {
2144 st->print("%s", Matcher::regName[dst_lo]);
2145 }
2146 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2147 int vsize = 0;
2148 switch (ideal_reg()) {
2149 case Op_VecD:
2150 vsize = 64;
2151 break;
2152 case Op_VecX:
2153 vsize = 128;
2154 break;
2155 case Op_VecA:
2156 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2157 break;
2158 default:
2159 assert(false, "bad register type for spill");
2160 ShouldNotReachHere();
2161 }
2162 st->print("\t# vector spill size = %d", vsize);
2163 } else if (ideal_reg() == Op_RegVectMask) {
2164 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2165 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2166 st->print("\t# predicate spill size = %d", vsize);
2167 } else {
2168 st->print("\t# spill size = %d", is64 ? 64 : 32);
2169 }
2170 }
2171
2172 return 0;
2173
2174 }
2175
2176 #ifndef PRODUCT
2177 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2178 if (!ra_)
2179 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2180 else
2181 implementation(nullptr, ra_, false, st);
2182 }
2183 #endif
2184
2185 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2186 implementation(masm, ra_, false, nullptr);
2187 }
2188
2189 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2190 return MachNode::size(ra_);
2191 }
2192
2193 //=============================================================================
2194
2195 #ifndef PRODUCT
2196 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2197 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2198 int reg = ra_->get_reg_first(this);
2199 st->print("add %s, rsp, #%d]\t# box lock",
2200 Matcher::regName[reg], offset);
2201 }
2202 #endif
2203
2204 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2205 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2206 int reg = ra_->get_encode(this);
2207
2208 // This add will handle any 24-bit signed offset. 24 bits allows an
2209 // 8 megabyte stack frame.
2210 __ add(as_Register(reg), sp, offset);
2211 }
2212
2213 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2214 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2215 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2216
2217 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2218 return NativeInstruction::instruction_size;
2219 } else {
2220 return 2 * NativeInstruction::instruction_size;
2221 }
2222 }
2223
2224 //=============================================================================
2225
2226 #ifndef PRODUCT
2227 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2228 {
2229 st->print_cr("# MachUEPNode");
2230 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2231 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2232 st->print_cr("\tcmpw rscratch1, r10");
2233 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2234 }
2235 #endif
2236
2237 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2238 {
2239 __ ic_check(InteriorEntryAlignment);
2240 }
2241
2242 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2243 {
2244 return MachNode::size(ra_);
2245 }
2246
2247 // REQUIRED EMIT CODE
2248
2249 //=============================================================================
2250
2251 // Emit deopt handler code.
2252 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2253 {
2254 // Note that the code buffer's insts_mark is always relative to insts.
2255 // That's why we must use the macroassembler to generate a handler.
2256 address base = __ start_a_stub(size_deopt_handler());
2257 if (base == nullptr) {
2258 ciEnv::current()->record_failure("CodeCache is full");
2259 return 0; // CodeBuffer::expand failed
2260 }
2261
2262 int offset = __ offset();
2263 Label start;
2264 __ bind(start);
2265 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2266
2267 int entry_offset = __ offset();
2268 __ b(start);
2269
2270 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2271 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2272 "out of bounds read in post-call NOP check");
2273 __ end_a_stub();
2274 return entry_offset;
2275 }
2276
2277 // REQUIRED MATCHER CODE
2278
2279 //=============================================================================
2280
2281 bool Matcher::match_rule_supported(int opcode) {
2282 if (!has_match_rule(opcode))
2283 return false;
2284
2285 switch (opcode) {
2286 case Op_OnSpinWait:
2287 return VM_Version::supports_on_spin_wait();
2288 case Op_CacheWB:
2289 case Op_CacheWBPreSync:
2290 case Op_CacheWBPostSync:
2291 if (!VM_Version::supports_data_cache_line_flush()) {
2292 return false;
2293 }
2294 break;
2295 case Op_ExpandBits:
2296 case Op_CompressBits:
2297 if (!VM_Version::supports_svebitperm()) {
2298 return false;
2299 }
2300 break;
2301 case Op_FmaF:
2302 case Op_FmaD:
2303 case Op_FmaVF:
2304 case Op_FmaVD:
2305 if (!UseFMA) {
2306 return false;
2307 }
2308 break;
2309 case Op_FmaHF:
2310 // UseFMA flag also needs to be checked along with FEAT_FP16
2311 if (!UseFMA || !is_feat_fp16_supported()) {
2312 return false;
2313 }
2314 break;
2315 case Op_AddHF:
2316 case Op_SubHF:
2317 case Op_MulHF:
2318 case Op_DivHF:
2319 case Op_MinHF:
2320 case Op_MaxHF:
2321 case Op_SqrtHF:
2322 // Half-precision floating point scalar operations require FEAT_FP16
2323 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2324 // features are supported.
2325 if (!is_feat_fp16_supported()) {
2326 return false;
2327 }
2328 break;
2329 }
2330
2331 return true; // Per default match rules are supported.
2332 }
2333
2334 const RegMask* Matcher::predicate_reg_mask(void) {
2335 return &_PR_REG_mask;
2336 }
2337
2338 bool Matcher::supports_vector_calling_convention(void) {
2339 return EnableVectorSupport;
2340 }
2341
2342 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2343 assert(EnableVectorSupport, "sanity");
2344 int lo = V0_num;
2345 int hi = V0_H_num;
2346 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2347 hi = V0_K_num;
2348 }
2349 return OptoRegPair(hi, lo);
2350 }
2351
2352 // Is this branch offset short enough that a short branch can be used?
2353 //
2354 // NOTE: If the platform does not provide any short branch variants, then
2355 // this method should return false for offset 0.
2356 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2357 // The passed offset is relative to address of the branch.
2358
2359 return (-32768 <= offset && offset < 32768);
2360 }
2361
2362 // Vector width in bytes.
2363 int Matcher::vector_width_in_bytes(BasicType bt) {
2364 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2365 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2366 // Minimum 2 values in vector
2367 if (size < 2*type2aelembytes(bt)) size = 0;
2368 // But never < 4
2369 if (size < 4) size = 0;
2370 return size;
2371 }
2372
2373 // Limits on vector size (number of elements) loaded into vector.
2374 int Matcher::max_vector_size(const BasicType bt) {
2375 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2376 }
2377
2378 int Matcher::min_vector_size(const BasicType bt) {
2379 // Usually, the shortest vector length supported by AArch64 ISA and
2380 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2381 // vectors in a few special cases.
2382 int size;
2383 switch(bt) {
2384 case T_BOOLEAN:
2385 // Load/store a vector mask with only 2 elements for vector types
2386 // such as "2I/2F/2L/2D".
2387 size = 2;
2388 break;
2389 case T_BYTE:
2390 // Generate a "4B" vector, to support vector cast between "8B/16B"
2391 // and "4S/4I/4L/4F/4D".
2392 size = 4;
2393 break;
2394 case T_SHORT:
2395 // Generate a "2S" vector, to support vector cast between "4S/8S"
2396 // and "2I/2L/2F/2D".
2397 size = 2;
2398 break;
2399 default:
2400 // Limit the min vector length to 64-bit.
2401 size = 8 / type2aelembytes(bt);
2402 // The number of elements in a vector should be at least 2.
2403 size = MAX2(size, 2);
2404 }
2405
2406 int max_size = max_vector_size(bt);
2407 return MIN2(size, max_size);
2408 }
2409
2410 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2411 return Matcher::max_vector_size(bt);
2412 }
2413
2414 // Actual max scalable vector register length.
2415 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2416 return Matcher::max_vector_size(bt);
2417 }
2418
2419 // Vector ideal reg.
2420 uint Matcher::vector_ideal_reg(int len) {
2421 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2422 return Op_VecA;
2423 }
2424 switch(len) {
2425 // For 16-bit/32-bit mask vector, reuse VecD.
2426 case 2:
2427 case 4:
2428 case 8: return Op_VecD;
2429 case 16: return Op_VecX;
2430 }
2431 ShouldNotReachHere();
2432 return 0;
2433 }
2434
2435 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2436 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2437 switch (ideal_reg) {
2438 case Op_VecA: return new vecAOper();
2439 case Op_VecD: return new vecDOper();
2440 case Op_VecX: return new vecXOper();
2441 }
2442 ShouldNotReachHere();
2443 return nullptr;
2444 }
2445
2446 bool Matcher::is_reg2reg_move(MachNode* m) {
2447 return false;
2448 }
2449
2450 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2451 return false;
2452 }
2453
2454 bool Matcher::is_generic_vector(MachOper* opnd) {
2455 return opnd->opcode() == VREG;
2456 }
2457
2458 #ifdef ASSERT
2459 // Return whether or not this register is ever used as an argument.
2460 bool Matcher::can_be_java_arg(int reg)
2461 {
2462 return
2463 reg == R0_num || reg == R0_H_num ||
2464 reg == R1_num || reg == R1_H_num ||
2465 reg == R2_num || reg == R2_H_num ||
2466 reg == R3_num || reg == R3_H_num ||
2467 reg == R4_num || reg == R4_H_num ||
2468 reg == R5_num || reg == R5_H_num ||
2469 reg == R6_num || reg == R6_H_num ||
2470 reg == R7_num || reg == R7_H_num ||
2471 reg == V0_num || reg == V0_H_num ||
2472 reg == V1_num || reg == V1_H_num ||
2473 reg == V2_num || reg == V2_H_num ||
2474 reg == V3_num || reg == V3_H_num ||
2475 reg == V4_num || reg == V4_H_num ||
2476 reg == V5_num || reg == V5_H_num ||
2477 reg == V6_num || reg == V6_H_num ||
2478 reg == V7_num || reg == V7_H_num;
2479 }
2480 #endif
2481
2482 uint Matcher::int_pressure_limit()
2483 {
2484 // JDK-8183543: When taking the number of available registers as int
2485 // register pressure threshold, the jtreg test:
2486 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2487 // failed due to C2 compilation failure with
2488 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2489 //
2490 // A derived pointer is live at CallNode and then is flagged by RA
2491 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2492 // derived pointers and lastly fail to spill after reaching maximum
2493 // number of iterations. Lowering the default pressure threshold to
2494 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2495 // a high register pressure area of the code so that split_DEF can
2496 // generate DefinitionSpillCopy for the derived pointer.
2497 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2498 if (!PreserveFramePointer) {
2499 // When PreserveFramePointer is off, frame pointer is allocatable,
2500 // but different from other SOC registers, it is excluded from
2501 // fatproj's mask because its save type is No-Save. Decrease 1 to
2502 // ensure high pressure at fatproj when PreserveFramePointer is off.
2503 // See check_pressure_at_fatproj().
2504 default_int_pressure_threshold--;
2505 }
2506 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2507 }
2508
2509 uint Matcher::float_pressure_limit()
2510 {
2511 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2512 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2513 }
2514
2515 const RegMask& Matcher::divI_proj_mask() {
2516 ShouldNotReachHere();
2517 return RegMask::EMPTY;
2518 }
2519
2520 // Register for MODI projection of divmodI.
2521 const RegMask& Matcher::modI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for DIVL projection of divmodL.
2527 const RegMask& Matcher::divL_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for MODL projection of divmodL.
2533 const RegMask& Matcher::modL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2539 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2540 Node* u = addp->fast_out(i);
2541 if (u->is_LoadStore()) {
2542 // On AArch64, LoadStoreNodes (i.e. compare and swap
2543 // instructions) only take register indirect as an operand, so
2544 // any attempt to use an AddPNode as an input to a LoadStoreNode
2545 // must fail.
2546 return false;
2547 }
2548 if (u->is_Mem()) {
2549 int opsize = u->as_Mem()->memory_size();
2550 assert(opsize > 0, "unexpected memory operand size");
2551 if (u->as_Mem()->memory_size() != (1<<shift)) {
2552 return false;
2553 }
2554 }
2555 }
2556 return true;
2557 }
2558
2559 // Convert BoolTest condition to Assembler condition.
2560 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2561 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2562 Assembler::Condition result;
2563 switch(cond) {
2564 case BoolTest::eq:
2565 result = Assembler::EQ; break;
2566 case BoolTest::ne:
2567 result = Assembler::NE; break;
2568 case BoolTest::le:
2569 result = Assembler::LE; break;
2570 case BoolTest::ge:
2571 result = Assembler::GE; break;
2572 case BoolTest::lt:
2573 result = Assembler::LT; break;
2574 case BoolTest::gt:
2575 result = Assembler::GT; break;
2576 case BoolTest::ule:
2577 result = Assembler::LS; break;
2578 case BoolTest::uge:
2579 result = Assembler::HS; break;
2580 case BoolTest::ult:
2581 result = Assembler::LO; break;
2582 case BoolTest::ugt:
2583 result = Assembler::HI; break;
2584 case BoolTest::overflow:
2585 result = Assembler::VS; break;
2586 case BoolTest::no_overflow:
2587 result = Assembler::VC; break;
2588 default:
2589 ShouldNotReachHere();
2590 return Assembler::Condition(-1);
2591 }
2592
2593 // Check conversion
2594 if (cond & BoolTest::unsigned_compare) {
2595 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2596 } else {
2597 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2598 }
2599
2600 return result;
2601 }
2602
2603 // Binary src (Replicate con)
2604 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2605 if (n == nullptr || m == nullptr) {
2606 return false;
2607 }
2608
2609 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2610 return false;
2611 }
2612
2613 Node* imm_node = m->in(1);
2614 if (!imm_node->is_Con()) {
2615 return false;
2616 }
2617
2618 const Type* t = imm_node->bottom_type();
2619 if (!(t->isa_int() || t->isa_long())) {
2620 return false;
2621 }
2622
2623 switch (n->Opcode()) {
2624 case Op_AndV:
2625 case Op_OrV:
2626 case Op_XorV: {
2627 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2628 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2629 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2630 }
2631 case Op_AddVB:
2632 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2633 case Op_AddVS:
2634 case Op_AddVI:
2635 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2636 case Op_AddVL:
2637 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2638 default:
2639 return false;
2640 }
2641 }
2642
2643 // (XorV src (Replicate m1))
2644 // (XorVMask src (MaskAll m1))
2645 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2646 if (n != nullptr && m != nullptr) {
2647 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2648 VectorNode::is_all_ones_vector(m);
2649 }
2650 return false;
2651 }
2652
2653 // Should the matcher clone input 'm' of node 'n'?
2654 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2655 if (is_vshift_con_pattern(n, m) ||
2656 is_vector_bitwise_not_pattern(n, m) ||
2657 is_valid_sve_arith_imm_pattern(n, m) ||
2658 is_encode_and_store_pattern(n, m)) {
2659 mstack.push(m, Visit);
2660 return true;
2661 }
2662 return false;
2663 }
2664
2665 // Should the Matcher clone shifts on addressing modes, expecting them
2666 // to be subsumed into complex addressing expressions or compute them
2667 // into registers?
2668 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2669
2670 // Loads and stores with indirect memory input (e.g., volatile loads and
2671 // stores) do not subsume the input into complex addressing expressions. If
2672 // the addressing expression is input to at least one such load or store, do
2673 // not clone the addressing expression. Query needs_acquiring_load and
2674 // needs_releasing_store as a proxy for indirect memory input, as it is not
2675 // possible to directly query for indirect memory input at this stage.
2676 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2677 Node* n = m->fast_out(i);
2678 if (n->is_Load() && needs_acquiring_load(n)) {
2679 return false;
2680 }
2681 if (n->is_Store() && needs_releasing_store(n)) {
2682 return false;
2683 }
2684 }
2685
2686 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2687 return true;
2688 }
2689
2690 Node *off = m->in(AddPNode::Offset);
2691 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2692 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2693 // Are there other uses besides address expressions?
2694 !is_visited(off)) {
2695 address_visited.set(off->_idx); // Flag as address_visited
2696 mstack.push(off->in(2), Visit);
2697 Node *conv = off->in(1);
2698 if (conv->Opcode() == Op_ConvI2L &&
2699 // Are there other uses besides address expressions?
2700 !is_visited(conv)) {
2701 address_visited.set(conv->_idx); // Flag as address_visited
2702 mstack.push(conv->in(1), Pre_Visit);
2703 } else {
2704 mstack.push(conv, Pre_Visit);
2705 }
2706 address_visited.test_set(m->_idx); // Flag as address_visited
2707 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2708 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2709 return true;
2710 } else if (off->Opcode() == Op_ConvI2L &&
2711 // Are there other uses besides address expressions?
2712 !is_visited(off)) {
2713 address_visited.test_set(m->_idx); // Flag as address_visited
2714 address_visited.set(off->_idx); // Flag as address_visited
2715 mstack.push(off->in(1), Pre_Visit);
2716 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2717 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2718 return true;
2719 }
2720 return false;
2721 }
2722
2723 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2724 { \
2725 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2726 guarantee(DISP == 0, "mode not permitted for volatile"); \
2727 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2728 __ INSN(REG, as_Register(BASE)); \
2729 }
2730
2731
2732 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2733 {
2734 Address::extend scale;
2735
2736 // Hooboy, this is fugly. We need a way to communicate to the
2737 // encoder that the index needs to be sign extended, so we have to
2738 // enumerate all the cases.
2739 switch (opcode) {
2740 case INDINDEXSCALEDI2L:
2741 case INDINDEXSCALEDI2LN:
2742 case INDINDEXI2L:
2743 case INDINDEXI2LN:
2744 scale = Address::sxtw(size);
2745 break;
2746 default:
2747 scale = Address::lsl(size);
2748 }
2749
2750 if (index == -1) {
2751 return Address(base, disp);
2752 } else {
2753 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2754 return Address(base, as_Register(index), scale);
2755 }
2756 }
2757
2758
2759 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2760 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2761 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2762 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2763 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2764
2765 // Used for all non-volatile memory accesses. The use of
2766 // $mem->opcode() to discover whether this pattern uses sign-extended
2767 // offsets is something of a kludge.
2768 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2769 Register reg, int opcode,
2770 Register base, int index, int scale, int disp,
2771 int size_in_memory)
2772 {
2773 Address addr = mem2address(opcode, base, index, scale, disp);
2774 if (addr.getMode() == Address::base_plus_offset) {
2775 /* Fix up any out-of-range offsets. */
2776 assert_different_registers(rscratch1, base);
2777 assert_different_registers(rscratch1, reg);
2778 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2779 }
2780 (masm->*insn)(reg, addr);
2781 }
2782
2783 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2784 FloatRegister reg, int opcode,
2785 Register base, int index, int size, int disp,
2786 int size_in_memory)
2787 {
2788 Address::extend scale;
2789
2790 switch (opcode) {
2791 case INDINDEXSCALEDI2L:
2792 case INDINDEXSCALEDI2LN:
2793 scale = Address::sxtw(size);
2794 break;
2795 default:
2796 scale = Address::lsl(size);
2797 }
2798
2799 if (index == -1) {
2800 // Fix up any out-of-range offsets.
2801 assert_different_registers(rscratch1, base);
2802 Address addr = Address(base, disp);
2803 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2804 (masm->*insn)(reg, addr);
2805 } else {
2806 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2807 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2808 }
2809 }
2810
2811 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2812 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2813 int opcode, Register base, int index, int size, int disp)
2814 {
2815 if (index == -1) {
2816 (masm->*insn)(reg, T, Address(base, disp));
2817 } else {
2818 assert(disp == 0, "unsupported address mode");
2819 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2820 }
2821 }
2822
2823 %}
2824
2825
2826
2827 //----------ENCODING BLOCK-----------------------------------------------------
2828 // This block specifies the encoding classes used by the compiler to
2829 // output byte streams. Encoding classes are parameterized macros
2830 // used by Machine Instruction Nodes in order to generate the bit
2831 // encoding of the instruction. Operands specify their base encoding
2832 // interface with the interface keyword. There are currently
2833 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2834 // COND_INTER. REG_INTER causes an operand to generate a function
2835 // which returns its register number when queried. CONST_INTER causes
2836 // an operand to generate a function which returns the value of the
2837 // constant when queried. MEMORY_INTER causes an operand to generate
2838 // four functions which return the Base Register, the Index Register,
2839 // the Scale Value, and the Offset Value of the operand when queried.
2840 // COND_INTER causes an operand to generate six functions which return
2841 // the encoding code (ie - encoding bits for the instruction)
2842 // associated with each basic boolean condition for a conditional
2843 // instruction.
2844 //
2845 // Instructions specify two basic values for encoding. Again, a
2846 // function is available to check if the constant displacement is an
2847 // oop. They use the ins_encode keyword to specify their encoding
2848 // classes (which must be a sequence of enc_class names, and their
2849 // parameters, specified in the encoding block), and they use the
2850 // opcode keyword to specify, in order, their primary, secondary, and
2851 // tertiary opcode. Only the opcode sections which a particular
2852 // instruction needs for encoding need to be specified.
2853 encode %{
2854 // Build emit functions for each basic byte or larger field in the
2855 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2856 // from C++ code in the enc_class source block. Emit functions will
2857 // live in the main source block for now. In future, we can
2858 // generalize this by adding a syntax that specifies the sizes of
2859 // fields in an order, so that the adlc can build the emit functions
2860 // automagically
2861
2862 // catch all for unimplemented encodings
2863 enc_class enc_unimplemented %{
2864 __ unimplemented("C2 catch all");
2865 %}
2866
2867 // BEGIN Non-volatile memory access
2868
2869 // This encoding class is generated automatically from ad_encode.m4.
2870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2871 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2872 Register dst_reg = as_Register($dst$$reg);
2873 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2874 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2875 %}
2876
2877 // This encoding class is generated automatically from ad_encode.m4.
2878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2879 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2880 Register dst_reg = as_Register($dst$$reg);
2881 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2883 %}
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2968 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2976 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2984 Register src_reg = as_Register($src$$reg);
2985 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_strb0(memory1 mem) %{
2992 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2993 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2994 %}
2995
2996 // This encoding class is generated automatically from ad_encode.m4.
2997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2998 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2999 Register src_reg = as_Register($src$$reg);
3000 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3001 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3002 %}
3003
3004 // This encoding class is generated automatically from ad_encode.m4.
3005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3006 enc_class aarch64_enc_strh0(memory2 mem) %{
3007 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3008 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3009 %}
3010
3011 // This encoding class is generated automatically from ad_encode.m4.
3012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3013 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3014 Register src_reg = as_Register($src$$reg);
3015 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strw0(memory4 mem) %{
3022 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3029 Register src_reg = as_Register($src$$reg);
3030 // we sometimes get asked to store the stack pointer into the
3031 // current thread -- we cannot do that directly on AArch64
3032 if (src_reg == r31_sp) {
3033 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3034 __ mov(rscratch2, sp);
3035 src_reg = rscratch2;
3036 }
3037 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_str0(memory8 mem) %{
3044 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3051 FloatRegister src_reg = as_FloatRegister($src$$reg);
3052 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3054 %}
3055
3056 // This encoding class is generated automatically from ad_encode.m4.
3057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3058 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3059 FloatRegister src_reg = as_FloatRegister($src$$reg);
3060 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3067 __ membar(Assembler::StoreStore);
3068 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3069 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3070 %}
3071
3072 // END Non-volatile memory access
3073
3074 // Vector loads and stores
3075 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3076 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3077 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3078 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3079 %}
3080
3081 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3082 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3083 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3084 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3085 %}
3086
3087 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3088 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3089 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3090 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3091 %}
3092
3093 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3100 FloatRegister src_reg = as_FloatRegister($src$$reg);
3101 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3106 FloatRegister src_reg = as_FloatRegister($src$$reg);
3107 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3112 FloatRegister src_reg = as_FloatRegister($src$$reg);
3113 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 // volatile loads and stores
3124
3125 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3126 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3127 rscratch1, stlrb);
3128 %}
3129
3130 enc_class aarch64_enc_stlrb0(memory mem) %{
3131 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3132 rscratch1, stlrb);
3133 %}
3134
3135 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3136 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, stlrh);
3138 %}
3139
3140 enc_class aarch64_enc_stlrh0(memory mem) %{
3141 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, stlrh);
3143 %}
3144
3145 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3146 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, stlrw);
3148 %}
3149
3150 enc_class aarch64_enc_stlrw0(memory mem) %{
3151 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3152 rscratch1, stlrw);
3153 %}
3154
3155 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3156 Register dst_reg = as_Register($dst$$reg);
3157 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, ldarb);
3159 __ sxtbw(dst_reg, dst_reg);
3160 %}
3161
3162 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3163 Register dst_reg = as_Register($dst$$reg);
3164 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, ldarb);
3166 __ sxtb(dst_reg, dst_reg);
3167 %}
3168
3169 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarb);
3172 %}
3173
3174 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarb);
3177 %}
3178
3179 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3180 Register dst_reg = as_Register($dst$$reg);
3181 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarh);
3183 __ sxthw(dst_reg, dst_reg);
3184 %}
3185
3186 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3187 Register dst_reg = as_Register($dst$$reg);
3188 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarh);
3190 __ sxth(dst_reg, dst_reg);
3191 %}
3192
3193 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3194 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarh);
3196 %}
3197
3198 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3199 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarh);
3201 %}
3202
3203 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3204 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarw);
3206 %}
3207
3208 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3209 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldarw);
3211 %}
3212
3213 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3214 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3215 rscratch1, ldar);
3216 %}
3217
3218 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3219 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3220 rscratch1, ldarw);
3221 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3222 %}
3223
3224 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3225 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldar);
3227 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3228 %}
3229
3230 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3231 Register src_reg = as_Register($src$$reg);
3232 // we sometimes get asked to store the stack pointer into the
3233 // current thread -- we cannot do that directly on AArch64
3234 if (src_reg == r31_sp) {
3235 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3236 __ mov(rscratch2, sp);
3237 src_reg = rscratch2;
3238 }
3239 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3240 rscratch1, stlr);
3241 %}
3242
3243 enc_class aarch64_enc_stlr0(memory mem) %{
3244 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3245 rscratch1, stlr);
3246 %}
3247
3248 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3249 {
3250 FloatRegister src_reg = as_FloatRegister($src$$reg);
3251 __ fmovs(rscratch2, src_reg);
3252 }
3253 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3254 rscratch1, stlrw);
3255 %}
3256
3257 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3258 {
3259 FloatRegister src_reg = as_FloatRegister($src$$reg);
3260 __ fmovd(rscratch2, src_reg);
3261 }
3262 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, stlr);
3264 %}
3265
3266 // synchronized read/update encodings
3267
3268 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3269 Register dst_reg = as_Register($dst$$reg);
3270 Register base = as_Register($mem$$base);
3271 int index = $mem$$index;
3272 int scale = $mem$$scale;
3273 int disp = $mem$$disp;
3274 if (index == -1) {
3275 if (disp != 0) {
3276 __ lea(rscratch1, Address(base, disp));
3277 __ ldaxr(dst_reg, rscratch1);
3278 } else {
3279 // TODO
3280 // should we ever get anything other than this case?
3281 __ ldaxr(dst_reg, base);
3282 }
3283 } else {
3284 Register index_reg = as_Register(index);
3285 if (disp == 0) {
3286 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3287 __ ldaxr(dst_reg, rscratch1);
3288 } else {
3289 __ lea(rscratch1, Address(base, disp));
3290 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3291 __ ldaxr(dst_reg, rscratch1);
3292 }
3293 }
3294 %}
3295
3296 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3297 Register src_reg = as_Register($src$$reg);
3298 Register base = as_Register($mem$$base);
3299 int index = $mem$$index;
3300 int scale = $mem$$scale;
3301 int disp = $mem$$disp;
3302 if (index == -1) {
3303 if (disp != 0) {
3304 __ lea(rscratch2, Address(base, disp));
3305 __ stlxr(rscratch1, src_reg, rscratch2);
3306 } else {
3307 // TODO
3308 // should we ever get anything other than this case?
3309 __ stlxr(rscratch1, src_reg, base);
3310 }
3311 } else {
3312 Register index_reg = as_Register(index);
3313 if (disp == 0) {
3314 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3315 __ stlxr(rscratch1, src_reg, rscratch2);
3316 } else {
3317 __ lea(rscratch2, Address(base, disp));
3318 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3319 __ stlxr(rscratch1, src_reg, rscratch2);
3320 }
3321 }
3322 __ cmpw(rscratch1, zr);
3323 %}
3324
3325 // prefetch encodings
3326
3327 enc_class aarch64_enc_prefetchw(memory mem) %{
3328 Register base = as_Register($mem$$base);
3329 int index = $mem$$index;
3330 int scale = $mem$$scale;
3331 int disp = $mem$$disp;
3332 if (index == -1) {
3333 // Fix up any out-of-range offsets.
3334 assert_different_registers(rscratch1, base);
3335 Address addr = Address(base, disp);
3336 addr = __ legitimize_address(addr, 8, rscratch1);
3337 __ prfm(addr, PSTL1KEEP);
3338 } else {
3339 Register index_reg = as_Register(index);
3340 if (disp == 0) {
3341 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3342 } else {
3343 __ lea(rscratch1, Address(base, disp));
3344 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3345 }
3346 }
3347 %}
3348
3349 // mov encodings
3350
3351 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3352 uint32_t con = (uint32_t)$src$$constant;
3353 Register dst_reg = as_Register($dst$$reg);
3354 if (con == 0) {
3355 __ movw(dst_reg, zr);
3356 } else {
3357 __ movw(dst_reg, con);
3358 }
3359 %}
3360
3361 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3362 Register dst_reg = as_Register($dst$$reg);
3363 uint64_t con = (uint64_t)$src$$constant;
3364 if (con == 0) {
3365 __ mov(dst_reg, zr);
3366 } else {
3367 __ mov(dst_reg, con);
3368 }
3369 %}
3370
3371 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3372 Register dst_reg = as_Register($dst$$reg);
3373 address con = (address)$src$$constant;
3374 if (con == nullptr || con == (address)1) {
3375 ShouldNotReachHere();
3376 } else {
3377 relocInfo::relocType rtype = $src->constant_reloc();
3378 if (rtype == relocInfo::oop_type) {
3379 __ movoop(dst_reg, (jobject)con);
3380 } else if (rtype == relocInfo::metadata_type) {
3381 __ mov_metadata(dst_reg, (Metadata*)con);
3382 } else {
3383 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3384 // load fake address constants using a normal move
3385 if (! __ is_valid_AArch64_address(con) ||
3386 con < (address)(uintptr_t)os::vm_page_size()) {
3387 __ mov(dst_reg, con);
3388 } else {
3389 // no reloc so just use adrp and add
3390 uint64_t offset;
3391 __ adrp(dst_reg, con, offset);
3392 __ add(dst_reg, dst_reg, offset);
3393 }
3394 }
3395 }
3396 %}
3397
3398 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3399 Register dst_reg = as_Register($dst$$reg);
3400 __ mov(dst_reg, zr);
3401 %}
3402
3403 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3404 Register dst_reg = as_Register($dst$$reg);
3405 __ mov(dst_reg, (uint64_t)1);
3406 %}
3407
3408 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3409 Register dst_reg = as_Register($dst$$reg);
3410 address con = (address)$src$$constant;
3411 if (con == nullptr) {
3412 ShouldNotReachHere();
3413 } else {
3414 relocInfo::relocType rtype = $src->constant_reloc();
3415 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3416 __ set_narrow_oop(dst_reg, (jobject)con);
3417 }
3418 %}
3419
3420 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3421 Register dst_reg = as_Register($dst$$reg);
3422 __ mov(dst_reg, zr);
3423 %}
3424
3425 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3426 Register dst_reg = as_Register($dst$$reg);
3427 address con = (address)$src$$constant;
3428 if (con == nullptr) {
3429 ShouldNotReachHere();
3430 } else {
3431 relocInfo::relocType rtype = $src->constant_reloc();
3432 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3433 __ set_narrow_klass(dst_reg, (Klass *)con);
3434 }
3435 %}
3436
3437 // arithmetic encodings
3438
3439 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 Register src_reg = as_Register($src1$$reg);
3442 int32_t con = (int32_t)$src2$$constant;
3443 // add has primary == 0, subtract has primary == 1
3444 if ($primary) { con = -con; }
3445 if (con < 0) {
3446 __ subw(dst_reg, src_reg, -con);
3447 } else {
3448 __ addw(dst_reg, src_reg, con);
3449 }
3450 %}
3451
3452 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 Register src_reg = as_Register($src1$$reg);
3455 int32_t con = (int32_t)$src2$$constant;
3456 // add has primary == 0, subtract has primary == 1
3457 if ($primary) { con = -con; }
3458 if (con < 0) {
3459 __ sub(dst_reg, src_reg, -con);
3460 } else {
3461 __ add(dst_reg, src_reg, con);
3462 }
3463 %}
3464
3465 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3466 Register dst_reg = as_Register($dst$$reg);
3467 Register src1_reg = as_Register($src1$$reg);
3468 Register src2_reg = as_Register($src2$$reg);
3469 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3470 %}
3471
3472 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3473 Register dst_reg = as_Register($dst$$reg);
3474 Register src1_reg = as_Register($src1$$reg);
3475 Register src2_reg = as_Register($src2$$reg);
3476 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3477 %}
3478
3479 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src1_reg = as_Register($src1$$reg);
3482 Register src2_reg = as_Register($src2$$reg);
3483 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3484 %}
3485
3486 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src1_reg = as_Register($src1$$reg);
3489 Register src2_reg = as_Register($src2$$reg);
3490 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3491 %}
3492
3493 // compare instruction encodings
3494
3495 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3496 Register reg1 = as_Register($src1$$reg);
3497 Register reg2 = as_Register($src2$$reg);
3498 __ cmpw(reg1, reg2);
3499 %}
3500
3501 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3502 Register reg = as_Register($src1$$reg);
3503 int32_t val = $src2$$constant;
3504 if (val >= 0) {
3505 __ subsw(zr, reg, val);
3506 } else {
3507 __ addsw(zr, reg, -val);
3508 }
3509 %}
3510
3511 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3512 Register reg1 = as_Register($src1$$reg);
3513 uint32_t val = (uint32_t)$src2$$constant;
3514 __ movw(rscratch1, val);
3515 __ cmpw(reg1, rscratch1);
3516 %}
3517
3518 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3519 Register reg1 = as_Register($src1$$reg);
3520 Register reg2 = as_Register($src2$$reg);
3521 __ cmp(reg1, reg2);
3522 %}
3523
3524 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3525 Register reg = as_Register($src1$$reg);
3526 int64_t val = $src2$$constant;
3527 if (val >= 0) {
3528 __ subs(zr, reg, val);
3529 } else if (val != -val) {
3530 __ adds(zr, reg, -val);
3531 } else {
3532 // aargh, Long.MIN_VALUE is a special case
3533 __ orr(rscratch1, zr, (uint64_t)val);
3534 __ subs(zr, reg, rscratch1);
3535 }
3536 %}
3537
3538 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3539 Register reg1 = as_Register($src1$$reg);
3540 uint64_t val = (uint64_t)$src2$$constant;
3541 __ mov(rscratch1, val);
3542 __ cmp(reg1, rscratch1);
3543 %}
3544
3545 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3546 Register reg1 = as_Register($src1$$reg);
3547 Register reg2 = as_Register($src2$$reg);
3548 __ cmp(reg1, reg2);
3549 %}
3550
3551 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3552 Register reg1 = as_Register($src1$$reg);
3553 Register reg2 = as_Register($src2$$reg);
3554 __ cmpw(reg1, reg2);
3555 %}
3556
3557 enc_class aarch64_enc_testp(iRegP src) %{
3558 Register reg = as_Register($src$$reg);
3559 __ cmp(reg, zr);
3560 %}
3561
3562 enc_class aarch64_enc_testn(iRegN src) %{
3563 Register reg = as_Register($src$$reg);
3564 __ cmpw(reg, zr);
3565 %}
3566
3567 enc_class aarch64_enc_b(label lbl) %{
3568 Label *L = $lbl$$label;
3569 __ b(*L);
3570 %}
3571
3572 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3573 Label *L = $lbl$$label;
3574 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3575 %}
3576
3577 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3578 Label *L = $lbl$$label;
3579 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3580 %}
3581
3582 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3583 %{
3584 Register sub_reg = as_Register($sub$$reg);
3585 Register super_reg = as_Register($super$$reg);
3586 Register temp_reg = as_Register($temp$$reg);
3587 Register result_reg = as_Register($result$$reg);
3588
3589 Label miss;
3590 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3591 nullptr, &miss,
3592 /*set_cond_codes:*/ true);
3593 if ($primary) {
3594 __ mov(result_reg, zr);
3595 }
3596 __ bind(miss);
3597 %}
3598
3599 enc_class aarch64_enc_java_static_call(method meth) %{
3600 address addr = (address)$meth$$method;
3601 address call;
3602 if (!_method) {
3603 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3604 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3605 if (call == nullptr) {
3606 ciEnv::current()->record_failure("CodeCache is full");
3607 return;
3608 }
3609 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3610 // The NOP here is purely to ensure that eliding a call to
3611 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3612 __ nop();
3613 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3614 } else {
3615 int method_index = resolved_method_index(masm);
3616 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3617 : static_call_Relocation::spec(method_index);
3618 call = __ trampoline_call(Address(addr, rspec));
3619 if (call == nullptr) {
3620 ciEnv::current()->record_failure("CodeCache is full");
3621 return;
3622 }
3623 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3624 // Calls of the same statically bound method can share
3625 // a stub to the interpreter.
3626 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3627 } else {
3628 // Emit stub for static call
3629 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3630 if (stub == nullptr) {
3631 ciEnv::current()->record_failure("CodeCache is full");
3632 return;
3633 }
3634 }
3635 }
3636
3637 __ post_call_nop();
3638
3639 // Only non uncommon_trap calls need to reinitialize ptrue.
3640 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3641 __ reinitialize_ptrue();
3642 }
3643 %}
3644
3645 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3646 int method_index = resolved_method_index(masm);
3647 address call = __ ic_call((address)$meth$$method, method_index);
3648 if (call == nullptr) {
3649 ciEnv::current()->record_failure("CodeCache is full");
3650 return;
3651 }
3652 __ post_call_nop();
3653 if (Compile::current()->max_vector_size() > 0) {
3654 __ reinitialize_ptrue();
3655 }
3656 %}
3657
3658 enc_class aarch64_enc_call_epilog() %{
3659 if (VerifyStackAtCalls) {
3660 // Check that stack depth is unchanged: find majik cookie on stack
3661 __ call_Unimplemented();
3662 }
3663 %}
3664
3665 enc_class aarch64_enc_java_to_runtime(method meth) %{
3666 // some calls to generated routines (arraycopy code) are scheduled
3667 // by C2 as runtime calls. if so we can call them using a br (they
3668 // will be in a reachable segment) otherwise we have to use a blr
3669 // which loads the absolute address into a register.
3670 address entry = (address)$meth$$method;
3671 CodeBlob *cb = CodeCache::find_blob(entry);
3672 if (cb) {
3673 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3674 if (call == nullptr) {
3675 ciEnv::current()->record_failure("CodeCache is full");
3676 return;
3677 }
3678 __ post_call_nop();
3679 } else {
3680 Label retaddr;
3681 // Make the anchor frame walkable
3682 __ adr(rscratch2, retaddr);
3683 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3684 __ lea(rscratch1, RuntimeAddress(entry));
3685 __ blr(rscratch1);
3686 __ bind(retaddr);
3687 __ post_call_nop();
3688 }
3689 if (Compile::current()->max_vector_size() > 0) {
3690 __ reinitialize_ptrue();
3691 }
3692 %}
3693
3694 enc_class aarch64_enc_rethrow() %{
3695 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3696 %}
3697
3698 enc_class aarch64_enc_ret() %{
3699 #ifdef ASSERT
3700 if (Compile::current()->max_vector_size() > 0) {
3701 __ verify_ptrue();
3702 }
3703 #endif
3704 __ ret(lr);
3705 %}
3706
3707 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3708 Register target_reg = as_Register($jump_target$$reg);
3709 __ br(target_reg);
3710 %}
3711
3712 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3713 Register target_reg = as_Register($jump_target$$reg);
3714 // exception oop should be in r0
3715 // ret addr has been popped into lr
3716 // callee expects it in r3
3717 __ mov(r3, lr);
3718 __ br(target_reg);
3719 %}
3720
3721 %}
3722
3723 //----------FRAME--------------------------------------------------------------
3724 // Definition of frame structure and management information.
3725 //
3726 // S T A C K L A Y O U T Allocators stack-slot number
3727 // | (to get allocators register number
3728 // G Owned by | | v add OptoReg::stack0())
3729 // r CALLER | |
3730 // o | +--------+ pad to even-align allocators stack-slot
3731 // w V | pad0 | numbers; owned by CALLER
3732 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3733 // h ^ | in | 5
3734 // | | args | 4 Holes in incoming args owned by SELF
3735 // | | | | 3
3736 // | | +--------+
3737 // V | | old out| Empty on Intel, window on Sparc
3738 // | old |preserve| Must be even aligned.
3739 // | SP-+--------+----> Matcher::_old_SP, even aligned
3740 // | | in | 3 area for Intel ret address
3741 // Owned by |preserve| Empty on Sparc.
3742 // SELF +--------+
3743 // | | pad2 | 2 pad to align old SP
3744 // | +--------+ 1
3745 // | | locks | 0
3746 // | +--------+----> OptoReg::stack0(), even aligned
3747 // | | pad1 | 11 pad to align new SP
3748 // | +--------+
3749 // | | | 10
3750 // | | spills | 9 spills
3751 // V | | 8 (pad0 slot for callee)
3752 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3753 // ^ | out | 7
3754 // | | args | 6 Holes in outgoing args owned by CALLEE
3755 // Owned by +--------+
3756 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3757 // | new |preserve| Must be even-aligned.
3758 // | SP-+--------+----> Matcher::_new_SP, even aligned
3759 // | | |
3760 //
3761 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3762 // known from SELF's arguments and the Java calling convention.
3763 // Region 6-7 is determined per call site.
3764 // Note 2: If the calling convention leaves holes in the incoming argument
3765 // area, those holes are owned by SELF. Holes in the outgoing area
3766 // are owned by the CALLEE. Holes should not be necessary in the
3767 // incoming area, as the Java calling convention is completely under
3768 // the control of the AD file. Doubles can be sorted and packed to
3769 // avoid holes. Holes in the outgoing arguments may be necessary for
3770 // varargs C calling conventions.
3771 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3772 // even aligned with pad0 as needed.
3773 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3774 // (the latter is true on Intel but is it false on AArch64?)
3775 // region 6-11 is even aligned; it may be padded out more so that
3776 // the region from SP to FP meets the minimum stack alignment.
3777 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3778 // alignment. Region 11, pad1, may be dynamically extended so that
3779 // SP meets the minimum alignment.
3780
3781 frame %{
3782 // These three registers define part of the calling convention
3783 // between compiled code and the interpreter.
3784
3785 // Inline Cache Register or Method for I2C.
3786 inline_cache_reg(R12);
3787
3788 // Number of stack slots consumed by locking an object
3789 sync_stack_slots(2);
3790
3791 // Compiled code's Frame Pointer
3792 frame_pointer(R31);
3793
3794 // Stack alignment requirement
3795 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3796
3797 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3798 // for calls to C. Supports the var-args backing area for register parms.
3799 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3800
3801 // The after-PROLOG location of the return address. Location of
3802 // return address specifies a type (REG or STACK) and a number
3803 // representing the register number (i.e. - use a register name) or
3804 // stack slot.
3805 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3806 // Otherwise, it is above the locks and verification slot and alignment word
3807 // TODO this may well be correct but need to check why that - 2 is there
3808 // ppc port uses 0 but we definitely need to allow for fixed_slots
3809 // which folds in the space used for monitors
3810 return_addr(STACK - 2 +
3811 align_up((Compile::current()->in_preserve_stack_slots() +
3812 Compile::current()->fixed_slots()),
3813 stack_alignment_in_slots()));
3814
3815 // Location of compiled Java return values. Same as C for now.
3816 return_value
3817 %{
3818 // TODO do we allow ideal_reg == Op_RegN???
3819 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3820 "only return normal values");
3821
3822 static const int lo[Op_RegL + 1] = { // enum name
3823 0, // Op_Node
3824 0, // Op_Set
3825 R0_num, // Op_RegN
3826 R0_num, // Op_RegI
3827 R0_num, // Op_RegP
3828 V0_num, // Op_RegF
3829 V0_num, // Op_RegD
3830 R0_num // Op_RegL
3831 };
3832
3833 static const int hi[Op_RegL + 1] = { // enum name
3834 0, // Op_Node
3835 0, // Op_Set
3836 OptoReg::Bad, // Op_RegN
3837 OptoReg::Bad, // Op_RegI
3838 R0_H_num, // Op_RegP
3839 OptoReg::Bad, // Op_RegF
3840 V0_H_num, // Op_RegD
3841 R0_H_num // Op_RegL
3842 };
3843
3844 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3845 %}
3846 %}
3847
3848 //----------ATTRIBUTES---------------------------------------------------------
3849 //----------Operand Attributes-------------------------------------------------
3850 op_attrib op_cost(1); // Required cost attribute
3851
3852 //----------Instruction Attributes---------------------------------------------
3853 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3854 ins_attrib ins_size(32); // Required size attribute (in bits)
3855 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3856 // a non-matching short branch variant
3857 // of some long branch?
3858 ins_attrib ins_alignment(4); // Required alignment attribute (must
3859 // be a power of 2) specifies the
3860 // alignment that some part of the
3861 // instruction (not necessarily the
3862 // start) requires. If > 1, a
3863 // compute_padding() function must be
3864 // provided for the instruction
3865
3866 // Whether this node is expanded during code emission into a sequence of
3867 // instructions and the first instruction can perform an implicit null check.
3868 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3869
3870 //----------OPERANDS-----------------------------------------------------------
3871 // Operand definitions must precede instruction definitions for correct parsing
3872 // in the ADLC because operands constitute user defined types which are used in
3873 // instruction definitions.
3874
3875 //----------Simple Operands----------------------------------------------------
3876
3877 // Integer operands 32 bit
3878 // 32 bit immediate
3879 operand immI()
3880 %{
3881 match(ConI);
3882
3883 op_cost(0);
3884 format %{ %}
3885 interface(CONST_INTER);
3886 %}
3887
3888 // 32 bit zero
3889 operand immI0()
3890 %{
3891 predicate(n->get_int() == 0);
3892 match(ConI);
3893
3894 op_cost(0);
3895 format %{ %}
3896 interface(CONST_INTER);
3897 %}
3898
3899 // 32 bit unit increment
3900 operand immI_1()
3901 %{
3902 predicate(n->get_int() == 1);
3903 match(ConI);
3904
3905 op_cost(0);
3906 format %{ %}
3907 interface(CONST_INTER);
3908 %}
3909
3910 // 32 bit unit decrement
3911 operand immI_M1()
3912 %{
3913 predicate(n->get_int() == -1);
3914 match(ConI);
3915
3916 op_cost(0);
3917 format %{ %}
3918 interface(CONST_INTER);
3919 %}
3920
3921 // Shift values for add/sub extension shift
3922 operand immIExt()
3923 %{
3924 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3925 match(ConI);
3926
3927 op_cost(0);
3928 format %{ %}
3929 interface(CONST_INTER);
3930 %}
3931
3932 operand immI_gt_1()
3933 %{
3934 predicate(n->get_int() > 1);
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 operand immI_le_4()
3943 %{
3944 predicate(n->get_int() <= 4);
3945 match(ConI);
3946
3947 op_cost(0);
3948 format %{ %}
3949 interface(CONST_INTER);
3950 %}
3951
3952 operand immI_16()
3953 %{
3954 predicate(n->get_int() == 16);
3955 match(ConI);
3956
3957 op_cost(0);
3958 format %{ %}
3959 interface(CONST_INTER);
3960 %}
3961
3962 operand immI_24()
3963 %{
3964 predicate(n->get_int() == 24);
3965 match(ConI);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 operand immI_32()
3973 %{
3974 predicate(n->get_int() == 32);
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_48()
3983 %{
3984 predicate(n->get_int() == 48);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_56()
3993 %{
3994 predicate(n->get_int() == 56);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_255()
4003 %{
4004 predicate(n->get_int() == 255);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_65535()
4013 %{
4014 predicate(n->get_int() == 65535);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_positive()
4023 %{
4024 predicate(n->get_int() > 0);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 // BoolTest condition for signed compare
4033 operand immI_cmp_cond()
4034 %{
4035 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4036 match(ConI);
4037
4038 op_cost(0);
4039 format %{ %}
4040 interface(CONST_INTER);
4041 %}
4042
4043 // BoolTest condition for unsigned compare
4044 operand immI_cmpU_cond()
4045 %{
4046 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4047 match(ConI);
4048
4049 op_cost(0);
4050 format %{ %}
4051 interface(CONST_INTER);
4052 %}
4053
4054 operand immL_255()
4055 %{
4056 predicate(n->get_long() == 255L);
4057 match(ConL);
4058
4059 op_cost(0);
4060 format %{ %}
4061 interface(CONST_INTER);
4062 %}
4063
4064 operand immL_65535()
4065 %{
4066 predicate(n->get_long() == 65535L);
4067 match(ConL);
4068
4069 op_cost(0);
4070 format %{ %}
4071 interface(CONST_INTER);
4072 %}
4073
4074 operand immL_4294967295()
4075 %{
4076 predicate(n->get_long() == 4294967295L);
4077 match(ConL);
4078
4079 op_cost(0);
4080 format %{ %}
4081 interface(CONST_INTER);
4082 %}
4083
4084 operand immL_bitmask()
4085 %{
4086 predicate((n->get_long() != 0)
4087 && ((n->get_long() & 0xc000000000000000l) == 0)
4088 && is_power_of_2(n->get_long() + 1));
4089 match(ConL);
4090
4091 op_cost(0);
4092 format %{ %}
4093 interface(CONST_INTER);
4094 %}
4095
4096 operand immI_bitmask()
4097 %{
4098 predicate((n->get_int() != 0)
4099 && ((n->get_int() & 0xc0000000) == 0)
4100 && is_power_of_2(n->get_int() + 1));
4101 match(ConI);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 operand immL_positive_bitmaskI()
4109 %{
4110 predicate((n->get_long() != 0)
4111 && ((julong)n->get_long() < 0x80000000ULL)
4112 && is_power_of_2(n->get_long() + 1));
4113 match(ConL);
4114
4115 op_cost(0);
4116 format %{ %}
4117 interface(CONST_INTER);
4118 %}
4119
4120 // Scale values for scaled offset addressing modes (up to long but not quad)
4121 operand immIScale()
4122 %{
4123 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4124 match(ConI);
4125
4126 op_cost(0);
4127 format %{ %}
4128 interface(CONST_INTER);
4129 %}
4130
4131 // 5 bit signed integer
4132 operand immI5()
4133 %{
4134 predicate(Assembler::is_simm(n->get_int(), 5));
4135 match(ConI);
4136
4137 op_cost(0);
4138 format %{ %}
4139 interface(CONST_INTER);
4140 %}
4141
4142 // 7 bit unsigned integer
4143 operand immIU7()
4144 %{
4145 predicate(Assembler::is_uimm(n->get_int(), 7));
4146 match(ConI);
4147
4148 op_cost(0);
4149 format %{ %}
4150 interface(CONST_INTER);
4151 %}
4152
4153 // Offset for scaled or unscaled immediate loads and stores
4154 operand immIOffset()
4155 %{
4156 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4157 match(ConI);
4158
4159 op_cost(0);
4160 format %{ %}
4161 interface(CONST_INTER);
4162 %}
4163
4164 operand immIOffset1()
4165 %{
4166 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4167 match(ConI);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 operand immIOffset2()
4175 %{
4176 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 operand immIOffset4()
4185 %{
4186 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immIOffset8()
4195 %{
4196 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4197 match(ConI);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 operand immIOffset16()
4205 %{
4206 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immLOffset()
4215 %{
4216 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4217 match(ConL);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immLoffset1()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4227 match(ConL);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immLoffset2()
4235 %{
4236 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4237 match(ConL);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immLoffset4()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4247 match(ConL);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immLoffset8()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4257 match(ConL);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immLoffset16()
4265 %{
4266 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4267 match(ConL);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 // 5 bit signed long integer
4275 operand immL5()
4276 %{
4277 predicate(Assembler::is_simm(n->get_long(), 5));
4278 match(ConL);
4279
4280 op_cost(0);
4281 format %{ %}
4282 interface(CONST_INTER);
4283 %}
4284
4285 // 7 bit unsigned long integer
4286 operand immLU7()
4287 %{
4288 predicate(Assembler::is_uimm(n->get_long(), 7));
4289 match(ConL);
4290
4291 op_cost(0);
4292 format %{ %}
4293 interface(CONST_INTER);
4294 %}
4295
4296 // 8 bit signed value.
4297 operand immI8()
4298 %{
4299 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4300 match(ConI);
4301
4302 op_cost(0);
4303 format %{ %}
4304 interface(CONST_INTER);
4305 %}
4306
4307 // 8 bit signed value (simm8), or #simm8 LSL 8.
4308 operand immIDupV()
4309 %{
4310 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4311 match(ConI);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 // 8 bit signed value (simm8), or #simm8 LSL 8.
4319 operand immLDupV()
4320 %{
4321 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4322 match(ConL);
4323
4324 op_cost(0);
4325 format %{ %}
4326 interface(CONST_INTER);
4327 %}
4328
4329 // 8 bit signed value (simm8), or #simm8 LSL 8.
4330 operand immHDupV()
4331 %{
4332 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4333 match(ConH);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 // 8 bit integer valid for vector add sub immediate
4341 operand immBAddSubV()
4342 %{
4343 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4344 match(ConI);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 // 32 bit integer valid for add sub immediate
4352 operand immIAddSub()
4353 %{
4354 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4355 match(ConI);
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 // 32 bit integer valid for vector add sub immediate
4362 operand immIAddSubV()
4363 %{
4364 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4365 match(ConI);
4366
4367 op_cost(0);
4368 format %{ %}
4369 interface(CONST_INTER);
4370 %}
4371
4372 // 32 bit unsigned integer valid for logical immediate
4373
4374 operand immBLog()
4375 %{
4376 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4377 match(ConI);
4378
4379 op_cost(0);
4380 format %{ %}
4381 interface(CONST_INTER);
4382 %}
4383
4384 operand immSLog()
4385 %{
4386 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4387 match(ConI);
4388
4389 op_cost(0);
4390 format %{ %}
4391 interface(CONST_INTER);
4392 %}
4393
4394 operand immILog()
4395 %{
4396 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4397 match(ConI);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 // Integer operands 64 bit
4405 // 64 bit immediate
4406 operand immL()
4407 %{
4408 match(ConL);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 // 64 bit zero
4416 operand immL0()
4417 %{
4418 predicate(n->get_long() == 0);
4419 match(ConL);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // 64 bit unit decrement
4427 operand immL_M1()
4428 %{
4429 predicate(n->get_long() == -1);
4430 match(ConL);
4431
4432 op_cost(0);
4433 format %{ %}
4434 interface(CONST_INTER);
4435 %}
4436
4437 // 64 bit integer valid for add sub immediate
4438 operand immLAddSub()
4439 %{
4440 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4441 match(ConL);
4442 op_cost(0);
4443 format %{ %}
4444 interface(CONST_INTER);
4445 %}
4446
4447 // 64 bit integer valid for addv subv immediate
4448 operand immLAddSubV()
4449 %{
4450 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4451 match(ConL);
4452
4453 op_cost(0);
4454 format %{ %}
4455 interface(CONST_INTER);
4456 %}
4457
4458 // 64 bit integer valid for logical immediate
4459 operand immLLog()
4460 %{
4461 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4462 match(ConL);
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 // Long Immediate: low 32-bit mask
4469 operand immL_32bits()
4470 %{
4471 predicate(n->get_long() == 0xFFFFFFFFL);
4472 match(ConL);
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // Pointer operands
4479 // Pointer Immediate
4480 operand immP()
4481 %{
4482 match(ConP);
4483
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 // nullptr Pointer Immediate
4490 operand immP0()
4491 %{
4492 predicate(n->get_ptr() == 0);
4493 match(ConP);
4494
4495 op_cost(0);
4496 format %{ %}
4497 interface(CONST_INTER);
4498 %}
4499
4500 // Pointer Immediate One
4501 // this is used in object initialization (initial object header)
4502 operand immP_1()
4503 %{
4504 predicate(n->get_ptr() == 1);
4505 match(ConP);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // AOT Runtime Constants Address
4513 operand immAOTRuntimeConstantsAddress()
4514 %{
4515 // Check if the address is in the range of AOT Runtime Constants
4516 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4517 match(ConP);
4518
4519 op_cost(0);
4520 format %{ %}
4521 interface(CONST_INTER);
4522 %}
4523
4524 // Float and Double operands
4525 // Double Immediate
4526 operand immD()
4527 %{
4528 match(ConD);
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // Double Immediate: +0.0d
4535 operand immD0()
4536 %{
4537 predicate(jlong_cast(n->getd()) == 0);
4538 match(ConD);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // constant 'double +0.0'.
4546 operand immDPacked()
4547 %{
4548 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4549 match(ConD);
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 // Float Immediate
4556 operand immF()
4557 %{
4558 match(ConF);
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 // Float Immediate: +0.0f.
4565 operand immF0()
4566 %{
4567 predicate(jint_cast(n->getf()) == 0);
4568 match(ConF);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // Half Float (FP16) Immediate
4576 operand immH()
4577 %{
4578 match(ConH);
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 //
4585 operand immFPacked()
4586 %{
4587 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4588 match(ConF);
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 // Narrow pointer operands
4595 // Narrow Pointer Immediate
4596 operand immN()
4597 %{
4598 match(ConN);
4599
4600 op_cost(0);
4601 format %{ %}
4602 interface(CONST_INTER);
4603 %}
4604
4605 // Narrow nullptr Pointer Immediate
4606 operand immN0()
4607 %{
4608 predicate(n->get_narrowcon() == 0);
4609 match(ConN);
4610
4611 op_cost(0);
4612 format %{ %}
4613 interface(CONST_INTER);
4614 %}
4615
4616 operand immNKlass()
4617 %{
4618 match(ConNKlass);
4619
4620 op_cost(0);
4621 format %{ %}
4622 interface(CONST_INTER);
4623 %}
4624
4625 // Integer 32 bit Register Operands
4626 // Integer 32 bitRegister (excludes SP)
4627 operand iRegI()
4628 %{
4629 constraint(ALLOC_IN_RC(any_reg32));
4630 match(RegI);
4631 match(iRegINoSp);
4632 op_cost(0);
4633 format %{ %}
4634 interface(REG_INTER);
4635 %}
4636
4637 // Integer 32 bit Register not Special
4638 operand iRegINoSp()
4639 %{
4640 constraint(ALLOC_IN_RC(no_special_reg32));
4641 match(RegI);
4642 op_cost(0);
4643 format %{ %}
4644 interface(REG_INTER);
4645 %}
4646
4647 // Integer 64 bit Register Operands
4648 // Integer 64 bit Register (includes SP)
4649 operand iRegL()
4650 %{
4651 constraint(ALLOC_IN_RC(any_reg));
4652 match(RegL);
4653 match(iRegLNoSp);
4654 op_cost(0);
4655 format %{ %}
4656 interface(REG_INTER);
4657 %}
4658
4659 // Integer 64 bit Register not Special
4660 operand iRegLNoSp()
4661 %{
4662 constraint(ALLOC_IN_RC(no_special_reg));
4663 match(RegL);
4664 match(iRegL_R0);
4665 format %{ %}
4666 interface(REG_INTER);
4667 %}
4668
4669 // Pointer Register Operands
4670 // Pointer Register
4671 operand iRegP()
4672 %{
4673 constraint(ALLOC_IN_RC(ptr_reg));
4674 match(RegP);
4675 match(iRegPNoSp);
4676 match(iRegP_R0);
4677 //match(iRegP_R2);
4678 //match(iRegP_R4);
4679 match(iRegP_R5);
4680 match(thread_RegP);
4681 op_cost(0);
4682 format %{ %}
4683 interface(REG_INTER);
4684 %}
4685
4686 // Pointer 64 bit Register not Special
4687 operand iRegPNoSp()
4688 %{
4689 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4690 match(RegP);
4691 // match(iRegP);
4692 // match(iRegP_R0);
4693 // match(iRegP_R2);
4694 // match(iRegP_R4);
4695 // match(iRegP_R5);
4696 // match(thread_RegP);
4697 op_cost(0);
4698 format %{ %}
4699 interface(REG_INTER);
4700 %}
4701
4702 // This operand is not allowed to use rfp even if
4703 // rfp is not used to hold the frame pointer.
4704 operand iRegPNoSpNoRfp()
4705 %{
4706 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4707 match(RegP);
4708 match(iRegPNoSp);
4709 op_cost(0);
4710 format %{ %}
4711 interface(REG_INTER);
4712 %}
4713
4714 // Pointer 64 bit Register R0 only
4715 operand iRegP_R0()
4716 %{
4717 constraint(ALLOC_IN_RC(r0_reg));
4718 match(RegP);
4719 // match(iRegP);
4720 match(iRegPNoSp);
4721 op_cost(0);
4722 format %{ %}
4723 interface(REG_INTER);
4724 %}
4725
4726 // Pointer 64 bit Register R1 only
4727 operand iRegP_R1()
4728 %{
4729 constraint(ALLOC_IN_RC(r1_reg));
4730 match(RegP);
4731 // match(iRegP);
4732 match(iRegPNoSp);
4733 op_cost(0);
4734 format %{ %}
4735 interface(REG_INTER);
4736 %}
4737
4738 // Pointer 64 bit Register R2 only
4739 operand iRegP_R2()
4740 %{
4741 constraint(ALLOC_IN_RC(r2_reg));
4742 match(RegP);
4743 // match(iRegP);
4744 match(iRegPNoSp);
4745 op_cost(0);
4746 format %{ %}
4747 interface(REG_INTER);
4748 %}
4749
4750 // Pointer 64 bit Register R3 only
4751 operand iRegP_R3()
4752 %{
4753 constraint(ALLOC_IN_RC(r3_reg));
4754 match(RegP);
4755 // match(iRegP);
4756 match(iRegPNoSp);
4757 op_cost(0);
4758 format %{ %}
4759 interface(REG_INTER);
4760 %}
4761
4762 // Pointer 64 bit Register R4 only
4763 operand iRegP_R4()
4764 %{
4765 constraint(ALLOC_IN_RC(r4_reg));
4766 match(RegP);
4767 // match(iRegP);
4768 match(iRegPNoSp);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // Pointer 64 bit Register R5 only
4775 operand iRegP_R5()
4776 %{
4777 constraint(ALLOC_IN_RC(r5_reg));
4778 match(RegP);
4779 // match(iRegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R10 only
4787 operand iRegP_R10()
4788 %{
4789 constraint(ALLOC_IN_RC(r10_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Long 64 bit Register R0 only
4799 operand iRegL_R0()
4800 %{
4801 constraint(ALLOC_IN_RC(r0_reg));
4802 match(RegL);
4803 match(iRegLNoSp);
4804 op_cost(0);
4805 format %{ %}
4806 interface(REG_INTER);
4807 %}
4808
4809 // Long 64 bit Register R11 only
4810 operand iRegL_R11()
4811 %{
4812 constraint(ALLOC_IN_RC(r11_reg));
4813 match(RegL);
4814 match(iRegLNoSp);
4815 op_cost(0);
4816 format %{ %}
4817 interface(REG_INTER);
4818 %}
4819
4820 // Register R0 only
4821 operand iRegI_R0()
4822 %{
4823 constraint(ALLOC_IN_RC(int_r0_reg));
4824 match(RegI);
4825 match(iRegINoSp);
4826 op_cost(0);
4827 format %{ %}
4828 interface(REG_INTER);
4829 %}
4830
4831 // Register R2 only
4832 operand iRegI_R2()
4833 %{
4834 constraint(ALLOC_IN_RC(int_r2_reg));
4835 match(RegI);
4836 match(iRegINoSp);
4837 op_cost(0);
4838 format %{ %}
4839 interface(REG_INTER);
4840 %}
4841
4842 // Register R3 only
4843 operand iRegI_R3()
4844 %{
4845 constraint(ALLOC_IN_RC(int_r3_reg));
4846 match(RegI);
4847 match(iRegINoSp);
4848 op_cost(0);
4849 format %{ %}
4850 interface(REG_INTER);
4851 %}
4852
4853
4854 // Register R4 only
4855 operand iRegI_R4()
4856 %{
4857 constraint(ALLOC_IN_RC(int_r4_reg));
4858 match(RegI);
4859 match(iRegINoSp);
4860 op_cost(0);
4861 format %{ %}
4862 interface(REG_INTER);
4863 %}
4864
4865
4866 // Pointer Register Operands
4867 // Narrow Pointer Register
4868 operand iRegN()
4869 %{
4870 constraint(ALLOC_IN_RC(any_reg32));
4871 match(RegN);
4872 match(iRegNNoSp);
4873 op_cost(0);
4874 format %{ %}
4875 interface(REG_INTER);
4876 %}
4877
4878 // Integer 64 bit Register not Special
4879 operand iRegNNoSp()
4880 %{
4881 constraint(ALLOC_IN_RC(no_special_reg32));
4882 match(RegN);
4883 op_cost(0);
4884 format %{ %}
4885 interface(REG_INTER);
4886 %}
4887
4888 // Float Register
4889 // Float register operands
4890 operand vRegF()
4891 %{
4892 constraint(ALLOC_IN_RC(float_reg));
4893 match(RegF);
4894
4895 op_cost(0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900 // Double Register
4901 // Double register operands
4902 operand vRegD()
4903 %{
4904 constraint(ALLOC_IN_RC(double_reg));
4905 match(RegD);
4906
4907 op_cost(0);
4908 format %{ %}
4909 interface(REG_INTER);
4910 %}
4911
4912 // Generic vector class. This will be used for
4913 // all vector operands, including NEON and SVE.
4914 operand vReg()
4915 %{
4916 constraint(ALLOC_IN_RC(dynamic));
4917 match(VecA);
4918 match(VecD);
4919 match(VecX);
4920
4921 op_cost(0);
4922 format %{ %}
4923 interface(REG_INTER);
4924 %}
4925
4926 operand vReg_V10()
4927 %{
4928 constraint(ALLOC_IN_RC(v10_veca_reg));
4929 match(vReg);
4930
4931 op_cost(0);
4932 format %{ %}
4933 interface(REG_INTER);
4934 %}
4935
4936 operand vReg_V11()
4937 %{
4938 constraint(ALLOC_IN_RC(v11_veca_reg));
4939 match(vReg);
4940
4941 op_cost(0);
4942 format %{ %}
4943 interface(REG_INTER);
4944 %}
4945
4946 operand vReg_V12()
4947 %{
4948 constraint(ALLOC_IN_RC(v12_veca_reg));
4949 match(vReg);
4950
4951 op_cost(0);
4952 format %{ %}
4953 interface(REG_INTER);
4954 %}
4955
4956 operand vReg_V13()
4957 %{
4958 constraint(ALLOC_IN_RC(v13_veca_reg));
4959 match(vReg);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 operand vReg_V17()
4967 %{
4968 constraint(ALLOC_IN_RC(v17_veca_reg));
4969 match(vReg);
4970
4971 op_cost(0);
4972 format %{ %}
4973 interface(REG_INTER);
4974 %}
4975
4976 operand vReg_V18()
4977 %{
4978 constraint(ALLOC_IN_RC(v18_veca_reg));
4979 match(vReg);
4980
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 operand vReg_V23()
4987 %{
4988 constraint(ALLOC_IN_RC(v23_veca_reg));
4989 match(vReg);
4990
4991 op_cost(0);
4992 format %{ %}
4993 interface(REG_INTER);
4994 %}
4995
4996 operand vReg_V24()
4997 %{
4998 constraint(ALLOC_IN_RC(v24_veca_reg));
4999 match(vReg);
5000
5001 op_cost(0);
5002 format %{ %}
5003 interface(REG_INTER);
5004 %}
5005
5006 operand vecA()
5007 %{
5008 constraint(ALLOC_IN_RC(vectora_reg));
5009 match(VecA);
5010
5011 op_cost(0);
5012 format %{ %}
5013 interface(REG_INTER);
5014 %}
5015
5016 operand vecD()
5017 %{
5018 constraint(ALLOC_IN_RC(vectord_reg));
5019 match(VecD);
5020
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 operand vecX()
5027 %{
5028 constraint(ALLOC_IN_RC(vectorx_reg));
5029 match(VecX);
5030
5031 op_cost(0);
5032 format %{ %}
5033 interface(REG_INTER);
5034 %}
5035
5036 operand vRegD_V0()
5037 %{
5038 constraint(ALLOC_IN_RC(v0_reg));
5039 match(RegD);
5040 op_cost(0);
5041 format %{ %}
5042 interface(REG_INTER);
5043 %}
5044
5045 operand vRegD_V1()
5046 %{
5047 constraint(ALLOC_IN_RC(v1_reg));
5048 match(RegD);
5049 op_cost(0);
5050 format %{ %}
5051 interface(REG_INTER);
5052 %}
5053
5054 operand vRegD_V2()
5055 %{
5056 constraint(ALLOC_IN_RC(v2_reg));
5057 match(RegD);
5058 op_cost(0);
5059 format %{ %}
5060 interface(REG_INTER);
5061 %}
5062
5063 operand vRegD_V3()
5064 %{
5065 constraint(ALLOC_IN_RC(v3_reg));
5066 match(RegD);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand vRegD_V4()
5073 %{
5074 constraint(ALLOC_IN_RC(v4_reg));
5075 match(RegD);
5076 op_cost(0);
5077 format %{ %}
5078 interface(REG_INTER);
5079 %}
5080
5081 operand vRegD_V5()
5082 %{
5083 constraint(ALLOC_IN_RC(v5_reg));
5084 match(RegD);
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vRegD_V6()
5091 %{
5092 constraint(ALLOC_IN_RC(v6_reg));
5093 match(RegD);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 operand vRegD_V7()
5100 %{
5101 constraint(ALLOC_IN_RC(v7_reg));
5102 match(RegD);
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 operand vRegD_V12()
5109 %{
5110 constraint(ALLOC_IN_RC(v12_reg));
5111 match(RegD);
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117 operand vRegD_V13()
5118 %{
5119 constraint(ALLOC_IN_RC(v13_reg));
5120 match(RegD);
5121 op_cost(0);
5122 format %{ %}
5123 interface(REG_INTER);
5124 %}
5125
5126 operand pReg()
5127 %{
5128 constraint(ALLOC_IN_RC(pr_reg));
5129 match(RegVectMask);
5130 match(pRegGov);
5131 op_cost(0);
5132 format %{ %}
5133 interface(REG_INTER);
5134 %}
5135
5136 operand pRegGov()
5137 %{
5138 constraint(ALLOC_IN_RC(gov_pr));
5139 match(RegVectMask);
5140 match(pReg);
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 operand pRegGov_P0()
5147 %{
5148 constraint(ALLOC_IN_RC(p0_reg));
5149 match(RegVectMask);
5150 op_cost(0);
5151 format %{ %}
5152 interface(REG_INTER);
5153 %}
5154
5155 operand pRegGov_P1()
5156 %{
5157 constraint(ALLOC_IN_RC(p1_reg));
5158 match(RegVectMask);
5159 op_cost(0);
5160 format %{ %}
5161 interface(REG_INTER);
5162 %}
5163
5164 // Flags register, used as output of signed compare instructions
5165
5166 // note that on AArch64 we also use this register as the output for
5167 // for floating point compare instructions (CmpF CmpD). this ensures
5168 // that ordered inequality tests use GT, GE, LT or LE none of which
5169 // pass through cases where the result is unordered i.e. one or both
5170 // inputs to the compare is a NaN. this means that the ideal code can
5171 // replace e.g. a GT with an LE and not end up capturing the NaN case
5172 // (where the comparison should always fail). EQ and NE tests are
5173 // always generated in ideal code so that unordered folds into the NE
5174 // case, matching the behaviour of AArch64 NE.
5175 //
5176 // This differs from x86 where the outputs of FP compares use a
5177 // special FP flags registers and where compares based on this
5178 // register are distinguished into ordered inequalities (cmpOpUCF) and
5179 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5180 // to explicitly handle the unordered case in branches. x86 also has
5181 // to include extra CMoveX rules to accept a cmpOpUCF input.
5182
5183 operand rFlagsReg()
5184 %{
5185 constraint(ALLOC_IN_RC(int_flags));
5186 match(RegFlags);
5187
5188 op_cost(0);
5189 format %{ "RFLAGS" %}
5190 interface(REG_INTER);
5191 %}
5192
5193 // Flags register, used as output of unsigned compare instructions
5194 operand rFlagsRegU()
5195 %{
5196 constraint(ALLOC_IN_RC(int_flags));
5197 match(RegFlags);
5198
5199 op_cost(0);
5200 format %{ "RFLAGSU" %}
5201 interface(REG_INTER);
5202 %}
5203
5204 // Special Registers
5205
5206 // Method Register
5207 operand inline_cache_RegP(iRegP reg)
5208 %{
5209 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5210 match(reg);
5211 match(iRegPNoSp);
5212 op_cost(0);
5213 format %{ %}
5214 interface(REG_INTER);
5215 %}
5216
5217 // Thread Register
5218 operand thread_RegP(iRegP reg)
5219 %{
5220 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5221 match(reg);
5222 op_cost(0);
5223 format %{ %}
5224 interface(REG_INTER);
5225 %}
5226
5227 //----------Memory Operands----------------------------------------------------
5228
5229 operand indirect(iRegP reg)
5230 %{
5231 constraint(ALLOC_IN_RC(ptr_reg));
5232 match(reg);
5233 op_cost(0);
5234 format %{ "[$reg]" %}
5235 interface(MEMORY_INTER) %{
5236 base($reg);
5237 index(0xffffffff);
5238 scale(0x0);
5239 disp(0x0);
5240 %}
5241 %}
5242
5243 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5244 %{
5245 constraint(ALLOC_IN_RC(ptr_reg));
5246 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5247 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5248 op_cost(0);
5249 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5250 interface(MEMORY_INTER) %{
5251 base($reg);
5252 index($ireg);
5253 scale($scale);
5254 disp(0x0);
5255 %}
5256 %}
5257
5258 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5259 %{
5260 constraint(ALLOC_IN_RC(ptr_reg));
5261 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5262 match(AddP reg (LShiftL lreg scale));
5263 op_cost(0);
5264 format %{ "$reg, $lreg lsl($scale)" %}
5265 interface(MEMORY_INTER) %{
5266 base($reg);
5267 index($lreg);
5268 scale($scale);
5269 disp(0x0);
5270 %}
5271 %}
5272
5273 operand indIndexI2L(iRegP reg, iRegI ireg)
5274 %{
5275 constraint(ALLOC_IN_RC(ptr_reg));
5276 match(AddP reg (ConvI2L ireg));
5277 op_cost(0);
5278 format %{ "$reg, $ireg, 0, I2L" %}
5279 interface(MEMORY_INTER) %{
5280 base($reg);
5281 index($ireg);
5282 scale(0x0);
5283 disp(0x0);
5284 %}
5285 %}
5286
5287 operand indIndex(iRegP reg, iRegL lreg)
5288 %{
5289 constraint(ALLOC_IN_RC(ptr_reg));
5290 match(AddP reg lreg);
5291 op_cost(0);
5292 format %{ "$reg, $lreg" %}
5293 interface(MEMORY_INTER) %{
5294 base($reg);
5295 index($lreg);
5296 scale(0x0);
5297 disp(0x0);
5298 %}
5299 %}
5300
5301 operand indOffI1(iRegP reg, immIOffset1 off)
5302 %{
5303 constraint(ALLOC_IN_RC(ptr_reg));
5304 match(AddP reg off);
5305 op_cost(0);
5306 format %{ "[$reg, $off]" %}
5307 interface(MEMORY_INTER) %{
5308 base($reg);
5309 index(0xffffffff);
5310 scale(0x0);
5311 disp($off);
5312 %}
5313 %}
5314
5315 operand indOffI2(iRegP reg, immIOffset2 off)
5316 %{
5317 constraint(ALLOC_IN_RC(ptr_reg));
5318 match(AddP reg off);
5319 op_cost(0);
5320 format %{ "[$reg, $off]" %}
5321 interface(MEMORY_INTER) %{
5322 base($reg);
5323 index(0xffffffff);
5324 scale(0x0);
5325 disp($off);
5326 %}
5327 %}
5328
5329 operand indOffI4(iRegP reg, immIOffset4 off)
5330 %{
5331 constraint(ALLOC_IN_RC(ptr_reg));
5332 match(AddP reg off);
5333 op_cost(0);
5334 format %{ "[$reg, $off]" %}
5335 interface(MEMORY_INTER) %{
5336 base($reg);
5337 index(0xffffffff);
5338 scale(0x0);
5339 disp($off);
5340 %}
5341 %}
5342
5343 operand indOffI8(iRegP reg, immIOffset8 off)
5344 %{
5345 constraint(ALLOC_IN_RC(ptr_reg));
5346 match(AddP reg off);
5347 op_cost(0);
5348 format %{ "[$reg, $off]" %}
5349 interface(MEMORY_INTER) %{
5350 base($reg);
5351 index(0xffffffff);
5352 scale(0x0);
5353 disp($off);
5354 %}
5355 %}
5356
5357 operand indOffI16(iRegP reg, immIOffset16 off)
5358 %{
5359 constraint(ALLOC_IN_RC(ptr_reg));
5360 match(AddP reg off);
5361 op_cost(0);
5362 format %{ "[$reg, $off]" %}
5363 interface(MEMORY_INTER) %{
5364 base($reg);
5365 index(0xffffffff);
5366 scale(0x0);
5367 disp($off);
5368 %}
5369 %}
5370
5371 operand indOffL1(iRegP reg, immLoffset1 off)
5372 %{
5373 constraint(ALLOC_IN_RC(ptr_reg));
5374 match(AddP reg off);
5375 op_cost(0);
5376 format %{ "[$reg, $off]" %}
5377 interface(MEMORY_INTER) %{
5378 base($reg);
5379 index(0xffffffff);
5380 scale(0x0);
5381 disp($off);
5382 %}
5383 %}
5384
5385 operand indOffL2(iRegP reg, immLoffset2 off)
5386 %{
5387 constraint(ALLOC_IN_RC(ptr_reg));
5388 match(AddP reg off);
5389 op_cost(0);
5390 format %{ "[$reg, $off]" %}
5391 interface(MEMORY_INTER) %{
5392 base($reg);
5393 index(0xffffffff);
5394 scale(0x0);
5395 disp($off);
5396 %}
5397 %}
5398
5399 operand indOffL4(iRegP reg, immLoffset4 off)
5400 %{
5401 constraint(ALLOC_IN_RC(ptr_reg));
5402 match(AddP reg off);
5403 op_cost(0);
5404 format %{ "[$reg, $off]" %}
5405 interface(MEMORY_INTER) %{
5406 base($reg);
5407 index(0xffffffff);
5408 scale(0x0);
5409 disp($off);
5410 %}
5411 %}
5412
5413 operand indOffL8(iRegP reg, immLoffset8 off)
5414 %{
5415 constraint(ALLOC_IN_RC(ptr_reg));
5416 match(AddP reg off);
5417 op_cost(0);
5418 format %{ "[$reg, $off]" %}
5419 interface(MEMORY_INTER) %{
5420 base($reg);
5421 index(0xffffffff);
5422 scale(0x0);
5423 disp($off);
5424 %}
5425 %}
5426
5427 operand indOffL16(iRegP reg, immLoffset16 off)
5428 %{
5429 constraint(ALLOC_IN_RC(ptr_reg));
5430 match(AddP reg off);
5431 op_cost(0);
5432 format %{ "[$reg, $off]" %}
5433 interface(MEMORY_INTER) %{
5434 base($reg);
5435 index(0xffffffff);
5436 scale(0x0);
5437 disp($off);
5438 %}
5439 %}
5440
5441 operand indirectX2P(iRegL reg)
5442 %{
5443 constraint(ALLOC_IN_RC(ptr_reg));
5444 match(CastX2P reg);
5445 op_cost(0);
5446 format %{ "[$reg]\t# long -> ptr" %}
5447 interface(MEMORY_INTER) %{
5448 base($reg);
5449 index(0xffffffff);
5450 scale(0x0);
5451 disp(0x0);
5452 %}
5453 %}
5454
5455 operand indOffX2P(iRegL reg, immLOffset off)
5456 %{
5457 constraint(ALLOC_IN_RC(ptr_reg));
5458 match(AddP (CastX2P reg) off);
5459 op_cost(0);
5460 format %{ "[$reg, $off]\t# long -> ptr" %}
5461 interface(MEMORY_INTER) %{
5462 base($reg);
5463 index(0xffffffff);
5464 scale(0x0);
5465 disp($off);
5466 %}
5467 %}
5468
5469 operand indirectN(iRegN reg)
5470 %{
5471 predicate(CompressedOops::shift() == 0);
5472 constraint(ALLOC_IN_RC(ptr_reg));
5473 match(DecodeN reg);
5474 op_cost(0);
5475 format %{ "[$reg]\t# narrow" %}
5476 interface(MEMORY_INTER) %{
5477 base($reg);
5478 index(0xffffffff);
5479 scale(0x0);
5480 disp(0x0);
5481 %}
5482 %}
5483
5484 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5485 %{
5486 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5487 constraint(ALLOC_IN_RC(ptr_reg));
5488 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5489 op_cost(0);
5490 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5491 interface(MEMORY_INTER) %{
5492 base($reg);
5493 index($ireg);
5494 scale($scale);
5495 disp(0x0);
5496 %}
5497 %}
5498
5499 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5500 %{
5501 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5502 constraint(ALLOC_IN_RC(ptr_reg));
5503 match(AddP (DecodeN reg) (LShiftL lreg scale));
5504 op_cost(0);
5505 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5506 interface(MEMORY_INTER) %{
5507 base($reg);
5508 index($lreg);
5509 scale($scale);
5510 disp(0x0);
5511 %}
5512 %}
5513
5514 operand indIndexI2LN(iRegN reg, iRegI ireg)
5515 %{
5516 predicate(CompressedOops::shift() == 0);
5517 constraint(ALLOC_IN_RC(ptr_reg));
5518 match(AddP (DecodeN reg) (ConvI2L ireg));
5519 op_cost(0);
5520 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5521 interface(MEMORY_INTER) %{
5522 base($reg);
5523 index($ireg);
5524 scale(0x0);
5525 disp(0x0);
5526 %}
5527 %}
5528
5529 operand indIndexN(iRegN reg, iRegL lreg)
5530 %{
5531 predicate(CompressedOops::shift() == 0);
5532 constraint(ALLOC_IN_RC(ptr_reg));
5533 match(AddP (DecodeN reg) lreg);
5534 op_cost(0);
5535 format %{ "$reg, $lreg\t# narrow" %}
5536 interface(MEMORY_INTER) %{
5537 base($reg);
5538 index($lreg);
5539 scale(0x0);
5540 disp(0x0);
5541 %}
5542 %}
5543
5544 operand indOffIN(iRegN reg, immIOffset off)
5545 %{
5546 predicate(CompressedOops::shift() == 0);
5547 constraint(ALLOC_IN_RC(ptr_reg));
5548 match(AddP (DecodeN reg) off);
5549 op_cost(0);
5550 format %{ "[$reg, $off]\t# narrow" %}
5551 interface(MEMORY_INTER) %{
5552 base($reg);
5553 index(0xffffffff);
5554 scale(0x0);
5555 disp($off);
5556 %}
5557 %}
5558
5559 operand indOffLN(iRegN reg, immLOffset off)
5560 %{
5561 predicate(CompressedOops::shift() == 0);
5562 constraint(ALLOC_IN_RC(ptr_reg));
5563 match(AddP (DecodeN reg) off);
5564 op_cost(0);
5565 format %{ "[$reg, $off]\t# narrow" %}
5566 interface(MEMORY_INTER) %{
5567 base($reg);
5568 index(0xffffffff);
5569 scale(0x0);
5570 disp($off);
5571 %}
5572 %}
5573
5574
5575 //----------Special Memory Operands--------------------------------------------
5576 // Stack Slot Operand - This operand is used for loading and storing temporary
5577 // values on the stack where a match requires a value to
5578 // flow through memory.
5579 operand stackSlotP(sRegP reg)
5580 %{
5581 constraint(ALLOC_IN_RC(stack_slots));
5582 op_cost(100);
5583 // No match rule because this operand is only generated in matching
5584 // match(RegP);
5585 format %{ "[$reg]" %}
5586 interface(MEMORY_INTER) %{
5587 base(0x1e); // RSP
5588 index(0x0); // No Index
5589 scale(0x0); // No Scale
5590 disp($reg); // Stack Offset
5591 %}
5592 %}
5593
5594 operand stackSlotI(sRegI reg)
5595 %{
5596 constraint(ALLOC_IN_RC(stack_slots));
5597 // No match rule because this operand is only generated in matching
5598 // match(RegI);
5599 format %{ "[$reg]" %}
5600 interface(MEMORY_INTER) %{
5601 base(0x1e); // RSP
5602 index(0x0); // No Index
5603 scale(0x0); // No Scale
5604 disp($reg); // Stack Offset
5605 %}
5606 %}
5607
5608 operand stackSlotF(sRegF reg)
5609 %{
5610 constraint(ALLOC_IN_RC(stack_slots));
5611 // No match rule because this operand is only generated in matching
5612 // match(RegF);
5613 format %{ "[$reg]" %}
5614 interface(MEMORY_INTER) %{
5615 base(0x1e); // RSP
5616 index(0x0); // No Index
5617 scale(0x0); // No Scale
5618 disp($reg); // Stack Offset
5619 %}
5620 %}
5621
5622 operand stackSlotD(sRegD reg)
5623 %{
5624 constraint(ALLOC_IN_RC(stack_slots));
5625 // No match rule because this operand is only generated in matching
5626 // match(RegD);
5627 format %{ "[$reg]" %}
5628 interface(MEMORY_INTER) %{
5629 base(0x1e); // RSP
5630 index(0x0); // No Index
5631 scale(0x0); // No Scale
5632 disp($reg); // Stack Offset
5633 %}
5634 %}
5635
5636 operand stackSlotL(sRegL reg)
5637 %{
5638 constraint(ALLOC_IN_RC(stack_slots));
5639 // No match rule because this operand is only generated in matching
5640 // match(RegL);
5641 format %{ "[$reg]" %}
5642 interface(MEMORY_INTER) %{
5643 base(0x1e); // RSP
5644 index(0x0); // No Index
5645 scale(0x0); // No Scale
5646 disp($reg); // Stack Offset
5647 %}
5648 %}
5649
5650 // Operands for expressing Control Flow
5651 // NOTE: Label is a predefined operand which should not be redefined in
5652 // the AD file. It is generically handled within the ADLC.
5653
5654 //----------Conditional Branch Operands----------------------------------------
5655 // Comparison Op - This is the operation of the comparison, and is limited to
5656 // the following set of codes:
5657 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5658 //
5659 // Other attributes of the comparison, such as unsignedness, are specified
5660 // by the comparison instruction that sets a condition code flags register.
5661 // That result is represented by a flags operand whose subtype is appropriate
5662 // to the unsignedness (etc.) of the comparison.
5663 //
5664 // Later, the instruction which matches both the Comparison Op (a Bool) and
5665 // the flags (produced by the Cmp) specifies the coding of the comparison op
5666 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5667
5668 // used for signed integral comparisons and fp comparisons
5669
5670 operand cmpOp()
5671 %{
5672 match(Bool);
5673
5674 format %{ "" %}
5675 interface(COND_INTER) %{
5676 equal(0x0, "eq");
5677 not_equal(0x1, "ne");
5678 less(0xb, "lt");
5679 greater_equal(0xa, "ge");
5680 less_equal(0xd, "le");
5681 greater(0xc, "gt");
5682 overflow(0x6, "vs");
5683 no_overflow(0x7, "vc");
5684 %}
5685 %}
5686
5687 // used for unsigned integral comparisons
5688
5689 operand cmpOpU()
5690 %{
5691 match(Bool);
5692
5693 format %{ "" %}
5694 interface(COND_INTER) %{
5695 equal(0x0, "eq");
5696 not_equal(0x1, "ne");
5697 less(0x3, "lo");
5698 greater_equal(0x2, "hs");
5699 less_equal(0x9, "ls");
5700 greater(0x8, "hi");
5701 overflow(0x6, "vs");
5702 no_overflow(0x7, "vc");
5703 %}
5704 %}
5705
5706 // used for certain integral comparisons which can be
5707 // converted to cbxx or tbxx instructions
5708
5709 operand cmpOpEqNe()
5710 %{
5711 match(Bool);
5712 op_cost(0);
5713 predicate(n->as_Bool()->_test._test == BoolTest::ne
5714 || n->as_Bool()->_test._test == BoolTest::eq);
5715
5716 format %{ "" %}
5717 interface(COND_INTER) %{
5718 equal(0x0, "eq");
5719 not_equal(0x1, "ne");
5720 less(0xb, "lt");
5721 greater_equal(0xa, "ge");
5722 less_equal(0xd, "le");
5723 greater(0xc, "gt");
5724 overflow(0x6, "vs");
5725 no_overflow(0x7, "vc");
5726 %}
5727 %}
5728
5729 // used for certain integral comparisons which can be
5730 // converted to cbxx or tbxx instructions
5731
5732 operand cmpOpLtGe()
5733 %{
5734 match(Bool);
5735 op_cost(0);
5736
5737 predicate(n->as_Bool()->_test._test == BoolTest::lt
5738 || n->as_Bool()->_test._test == BoolTest::ge);
5739
5740 format %{ "" %}
5741 interface(COND_INTER) %{
5742 equal(0x0, "eq");
5743 not_equal(0x1, "ne");
5744 less(0xb, "lt");
5745 greater_equal(0xa, "ge");
5746 less_equal(0xd, "le");
5747 greater(0xc, "gt");
5748 overflow(0x6, "vs");
5749 no_overflow(0x7, "vc");
5750 %}
5751 %}
5752
5753 // used for certain unsigned integral comparisons which can be
5754 // converted to cbxx or tbxx instructions
5755
5756 operand cmpOpUEqNeLeGt()
5757 %{
5758 match(Bool);
5759 op_cost(0);
5760
5761 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5762 n->as_Bool()->_test._test == BoolTest::ne ||
5763 n->as_Bool()->_test._test == BoolTest::le ||
5764 n->as_Bool()->_test._test == BoolTest::gt);
5765
5766 format %{ "" %}
5767 interface(COND_INTER) %{
5768 equal(0x0, "eq");
5769 not_equal(0x1, "ne");
5770 less(0x3, "lo");
5771 greater_equal(0x2, "hs");
5772 less_equal(0x9, "ls");
5773 greater(0x8, "hi");
5774 overflow(0x6, "vs");
5775 no_overflow(0x7, "vc");
5776 %}
5777 %}
5778
5779 // Special operand allowing long args to int ops to be truncated for free
5780
5781 operand iRegL2I(iRegL reg) %{
5782
5783 op_cost(0);
5784
5785 match(ConvL2I reg);
5786
5787 format %{ "l2i($reg)" %}
5788
5789 interface(REG_INTER)
5790 %}
5791
5792 operand iRegL2P(iRegL reg) %{
5793
5794 op_cost(0);
5795
5796 match(CastX2P reg);
5797
5798 format %{ "l2p($reg)" %}
5799
5800 interface(REG_INTER)
5801 %}
5802
5803 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5804 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5805 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5806 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5807
5808 //----------OPERAND CLASSES----------------------------------------------------
5809 // Operand Classes are groups of operands that are used as to simplify
5810 // instruction definitions by not requiring the AD writer to specify
5811 // separate instructions for every form of operand when the
5812 // instruction accepts multiple operand types with the same basic
5813 // encoding and format. The classic case of this is memory operands.
5814
5815 // memory is used to define read/write location for load/store
5816 // instruction defs. we can turn a memory op into an Address
5817
5818 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5819 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5820
5821 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5822 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5823
5824 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5825 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5826
5827 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5828 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5829
5830 // All of the memory operands. For the pipeline description.
5831 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5832 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5833 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5834
5835
5836 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5837 // operations. it allows the src to be either an iRegI or a (ConvL2I
5838 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5839 // can be elided because the 32-bit instruction will just employ the
5840 // lower 32 bits anyway.
5841 //
5842 // n.b. this does not elide all L2I conversions. if the truncated
5843 // value is consumed by more than one operation then the ConvL2I
5844 // cannot be bundled into the consuming nodes so an l2i gets planted
5845 // (actually a movw $dst $src) and the downstream instructions consume
5846 // the result of the l2i as an iRegI input. That's a shame since the
5847 // movw is actually redundant but its not too costly.
5848
5849 opclass iRegIorL2I(iRegI, iRegL2I);
5850 opclass iRegPorL2P(iRegP, iRegL2P);
5851
5852 //----------PIPELINE-----------------------------------------------------------
5853 // Rules which define the behavior of the target architectures pipeline.
5854
5855 // For specific pipelines, eg A53, define the stages of that pipeline
5856 //pipe_desc(ISS, EX1, EX2, WR);
5857 #define ISS S0
5858 #define EX1 S1
5859 #define EX2 S2
5860 #define WR S3
5861
5862 // Integer ALU reg operation
5863 pipeline %{
5864
5865 attributes %{
5866 // ARM instructions are of fixed length
5867 fixed_size_instructions; // Fixed size instructions TODO does
5868 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5869 // ARM instructions come in 32-bit word units
5870 instruction_unit_size = 4; // An instruction is 4 bytes long
5871 instruction_fetch_unit_size = 64; // The processor fetches one line
5872 instruction_fetch_units = 1; // of 64 bytes
5873 %}
5874
5875 // We don't use an actual pipeline model so don't care about resources
5876 // or description. we do use pipeline classes to introduce fixed
5877 // latencies
5878
5879 //----------RESOURCES----------------------------------------------------------
5880 // Resources are the functional units available to the machine
5881
5882 resources( INS0, INS1, INS01 = INS0 | INS1,
5883 ALU0, ALU1, ALU = ALU0 | ALU1,
5884 MAC,
5885 DIV,
5886 BRANCH,
5887 LDST,
5888 NEON_FP);
5889
5890 //----------PIPELINE DESCRIPTION-----------------------------------------------
5891 // Pipeline Description specifies the stages in the machine's pipeline
5892
5893 // Define the pipeline as a generic 6 stage pipeline
5894 pipe_desc(S0, S1, S2, S3, S4, S5);
5895
5896 //----------PIPELINE CLASSES---------------------------------------------------
5897 // Pipeline Classes describe the stages in which input and output are
5898 // referenced by the hardware pipeline.
5899
5900 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5901 %{
5902 single_instruction;
5903 src1 : S1(read);
5904 src2 : S2(read);
5905 dst : S5(write);
5906 INS01 : ISS;
5907 NEON_FP : S5;
5908 %}
5909
5910 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5911 %{
5912 single_instruction;
5913 src1 : S1(read);
5914 src2 : S2(read);
5915 dst : S5(write);
5916 INS01 : ISS;
5917 NEON_FP : S5;
5918 %}
5919
5920 pipe_class fp_uop_s(vRegF dst, vRegF src)
5921 %{
5922 single_instruction;
5923 src : S1(read);
5924 dst : S5(write);
5925 INS01 : ISS;
5926 NEON_FP : S5;
5927 %}
5928
5929 pipe_class fp_uop_d(vRegD dst, vRegD src)
5930 %{
5931 single_instruction;
5932 src : S1(read);
5933 dst : S5(write);
5934 INS01 : ISS;
5935 NEON_FP : S5;
5936 %}
5937
5938 pipe_class fp_d2f(vRegF dst, vRegD src)
5939 %{
5940 single_instruction;
5941 src : S1(read);
5942 dst : S5(write);
5943 INS01 : ISS;
5944 NEON_FP : S5;
5945 %}
5946
5947 pipe_class fp_f2d(vRegD dst, vRegF src)
5948 %{
5949 single_instruction;
5950 src : S1(read);
5951 dst : S5(write);
5952 INS01 : ISS;
5953 NEON_FP : S5;
5954 %}
5955
5956 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5957 %{
5958 single_instruction;
5959 src : S1(read);
5960 dst : S5(write);
5961 INS01 : ISS;
5962 NEON_FP : S5;
5963 %}
5964
5965 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5966 %{
5967 single_instruction;
5968 src : S1(read);
5969 dst : S5(write);
5970 INS01 : ISS;
5971 NEON_FP : S5;
5972 %}
5973
5974 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5975 %{
5976 single_instruction;
5977 src : S1(read);
5978 dst : S5(write);
5979 INS01 : ISS;
5980 NEON_FP : S5;
5981 %}
5982
5983 pipe_class fp_l2f(vRegF dst, iRegL src)
5984 %{
5985 single_instruction;
5986 src : S1(read);
5987 dst : S5(write);
5988 INS01 : ISS;
5989 NEON_FP : S5;
5990 %}
5991
5992 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5993 %{
5994 single_instruction;
5995 src : S1(read);
5996 dst : S5(write);
5997 INS01 : ISS;
5998 NEON_FP : S5;
5999 %}
6000
6001 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6002 %{
6003 single_instruction;
6004 src : S1(read);
6005 dst : S5(write);
6006 INS01 : ISS;
6007 NEON_FP : S5;
6008 %}
6009
6010 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6011 %{
6012 single_instruction;
6013 src : S1(read);
6014 dst : S5(write);
6015 INS01 : ISS;
6016 NEON_FP : S5;
6017 %}
6018
6019 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6020 %{
6021 single_instruction;
6022 src : S1(read);
6023 dst : S5(write);
6024 INS01 : ISS;
6025 NEON_FP : S5;
6026 %}
6027
6028 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6029 %{
6030 single_instruction;
6031 src1 : S1(read);
6032 src2 : S2(read);
6033 dst : S5(write);
6034 INS0 : ISS;
6035 NEON_FP : S5;
6036 %}
6037
6038 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6039 %{
6040 single_instruction;
6041 src1 : S1(read);
6042 src2 : S2(read);
6043 dst : S5(write);
6044 INS0 : ISS;
6045 NEON_FP : S5;
6046 %}
6047
6048 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6049 %{
6050 single_instruction;
6051 cr : S1(read);
6052 src1 : S1(read);
6053 src2 : S1(read);
6054 dst : S3(write);
6055 INS01 : ISS;
6056 NEON_FP : S3;
6057 %}
6058
6059 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6060 %{
6061 single_instruction;
6062 cr : S1(read);
6063 src1 : S1(read);
6064 src2 : S1(read);
6065 dst : S3(write);
6066 INS01 : ISS;
6067 NEON_FP : S3;
6068 %}
6069
6070 pipe_class fp_imm_s(vRegF dst)
6071 %{
6072 single_instruction;
6073 dst : S3(write);
6074 INS01 : ISS;
6075 NEON_FP : S3;
6076 %}
6077
6078 pipe_class fp_imm_d(vRegD dst)
6079 %{
6080 single_instruction;
6081 dst : S3(write);
6082 INS01 : ISS;
6083 NEON_FP : S3;
6084 %}
6085
6086 pipe_class fp_load_constant_s(vRegF dst)
6087 %{
6088 single_instruction;
6089 dst : S4(write);
6090 INS01 : ISS;
6091 NEON_FP : S4;
6092 %}
6093
6094 pipe_class fp_load_constant_d(vRegD dst)
6095 %{
6096 single_instruction;
6097 dst : S4(write);
6098 INS01 : ISS;
6099 NEON_FP : S4;
6100 %}
6101
6102 //------- Integer ALU operations --------------------------
6103
6104 // Integer ALU reg-reg operation
6105 // Operands needed in EX1, result generated in EX2
6106 // Eg. ADD x0, x1, x2
6107 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6108 %{
6109 single_instruction;
6110 dst : EX2(write);
6111 src1 : EX1(read);
6112 src2 : EX1(read);
6113 INS01 : ISS; // Dual issue as instruction 0 or 1
6114 ALU : EX2;
6115 %}
6116
6117 // Integer ALU reg-reg operation with constant shift
6118 // Shifted register must be available in LATE_ISS instead of EX1
6119 // Eg. ADD x0, x1, x2, LSL #2
6120 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6121 %{
6122 single_instruction;
6123 dst : EX2(write);
6124 src1 : EX1(read);
6125 src2 : ISS(read);
6126 INS01 : ISS;
6127 ALU : EX2;
6128 %}
6129
6130 // Integer ALU reg operation with constant shift
6131 // Eg. LSL x0, x1, #shift
6132 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6133 %{
6134 single_instruction;
6135 dst : EX2(write);
6136 src1 : ISS(read);
6137 INS01 : ISS;
6138 ALU : EX2;
6139 %}
6140
6141 // Integer ALU reg-reg operation with variable shift
6142 // Both operands must be available in LATE_ISS instead of EX1
6143 // Result is available in EX1 instead of EX2
6144 // Eg. LSLV x0, x1, x2
6145 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6146 %{
6147 single_instruction;
6148 dst : EX1(write);
6149 src1 : ISS(read);
6150 src2 : ISS(read);
6151 INS01 : ISS;
6152 ALU : EX1;
6153 %}
6154
6155 // Integer ALU reg-reg operation with extract
6156 // As for _vshift above, but result generated in EX2
6157 // Eg. EXTR x0, x1, x2, #N
6158 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6159 %{
6160 single_instruction;
6161 dst : EX2(write);
6162 src1 : ISS(read);
6163 src2 : ISS(read);
6164 INS1 : ISS; // Can only dual issue as Instruction 1
6165 ALU : EX1;
6166 %}
6167
6168 // Integer ALU reg operation
6169 // Eg. NEG x0, x1
6170 pipe_class ialu_reg(iRegI dst, iRegI src)
6171 %{
6172 single_instruction;
6173 dst : EX2(write);
6174 src : EX1(read);
6175 INS01 : ISS;
6176 ALU : EX2;
6177 %}
6178
6179 // Integer ALU reg mmediate operation
6180 // Eg. ADD x0, x1, #N
6181 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6182 %{
6183 single_instruction;
6184 dst : EX2(write);
6185 src1 : EX1(read);
6186 INS01 : ISS;
6187 ALU : EX2;
6188 %}
6189
6190 // Integer ALU immediate operation (no source operands)
6191 // Eg. MOV x0, #N
6192 pipe_class ialu_imm(iRegI dst)
6193 %{
6194 single_instruction;
6195 dst : EX1(write);
6196 INS01 : ISS;
6197 ALU : EX1;
6198 %}
6199
6200 //------- Compare operation -------------------------------
6201
6202 // Compare reg-reg
6203 // Eg. CMP x0, x1
6204 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6205 %{
6206 single_instruction;
6207 // fixed_latency(16);
6208 cr : EX2(write);
6209 op1 : EX1(read);
6210 op2 : EX1(read);
6211 INS01 : ISS;
6212 ALU : EX2;
6213 %}
6214
6215 // Compare reg-reg
6216 // Eg. CMP x0, #N
6217 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6218 %{
6219 single_instruction;
6220 // fixed_latency(16);
6221 cr : EX2(write);
6222 op1 : EX1(read);
6223 INS01 : ISS;
6224 ALU : EX2;
6225 %}
6226
6227 //------- Conditional instructions ------------------------
6228
6229 // Conditional no operands
6230 // Eg. CSINC x0, zr, zr, <cond>
6231 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6232 %{
6233 single_instruction;
6234 cr : EX1(read);
6235 dst : EX2(write);
6236 INS01 : ISS;
6237 ALU : EX2;
6238 %}
6239
6240 // Conditional 2 operand
6241 // EG. CSEL X0, X1, X2, <cond>
6242 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6243 %{
6244 single_instruction;
6245 cr : EX1(read);
6246 src1 : EX1(read);
6247 src2 : EX1(read);
6248 dst : EX2(write);
6249 INS01 : ISS;
6250 ALU : EX2;
6251 %}
6252
6253 // Conditional 2 operand
6254 // EG. CSEL X0, X1, X2, <cond>
6255 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6256 %{
6257 single_instruction;
6258 cr : EX1(read);
6259 src : EX1(read);
6260 dst : EX2(write);
6261 INS01 : ISS;
6262 ALU : EX2;
6263 %}
6264
6265 //------- Multiply pipeline operations --------------------
6266
6267 // Multiply reg-reg
6268 // Eg. MUL w0, w1, w2
6269 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6270 %{
6271 single_instruction;
6272 dst : WR(write);
6273 src1 : ISS(read);
6274 src2 : ISS(read);
6275 INS01 : ISS;
6276 MAC : WR;
6277 %}
6278
6279 // Multiply accumulate
6280 // Eg. MADD w0, w1, w2, w3
6281 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6282 %{
6283 single_instruction;
6284 dst : WR(write);
6285 src1 : ISS(read);
6286 src2 : ISS(read);
6287 src3 : ISS(read);
6288 INS01 : ISS;
6289 MAC : WR;
6290 %}
6291
6292 // Eg. MUL w0, w1, w2
6293 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6294 %{
6295 single_instruction;
6296 fixed_latency(3); // Maximum latency for 64 bit mul
6297 dst : WR(write);
6298 src1 : ISS(read);
6299 src2 : ISS(read);
6300 INS01 : ISS;
6301 MAC : WR;
6302 %}
6303
6304 // Multiply accumulate
6305 // Eg. MADD w0, w1, w2, w3
6306 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6307 %{
6308 single_instruction;
6309 fixed_latency(3); // Maximum latency for 64 bit mul
6310 dst : WR(write);
6311 src1 : ISS(read);
6312 src2 : ISS(read);
6313 src3 : ISS(read);
6314 INS01 : ISS;
6315 MAC : WR;
6316 %}
6317
6318 //------- Divide pipeline operations --------------------
6319
6320 // Eg. SDIV w0, w1, w2
6321 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6322 %{
6323 single_instruction;
6324 fixed_latency(8); // Maximum latency for 32 bit divide
6325 dst : WR(write);
6326 src1 : ISS(read);
6327 src2 : ISS(read);
6328 INS0 : ISS; // Can only dual issue as instruction 0
6329 DIV : WR;
6330 %}
6331
6332 // Eg. SDIV x0, x1, x2
6333 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6334 %{
6335 single_instruction;
6336 fixed_latency(16); // Maximum latency for 64 bit divide
6337 dst : WR(write);
6338 src1 : ISS(read);
6339 src2 : ISS(read);
6340 INS0 : ISS; // Can only dual issue as instruction 0
6341 DIV : WR;
6342 %}
6343
6344 //------- Load pipeline operations ------------------------
6345
6346 // Load - prefetch
6347 // Eg. PFRM <mem>
6348 pipe_class iload_prefetch(memory mem)
6349 %{
6350 single_instruction;
6351 mem : ISS(read);
6352 INS01 : ISS;
6353 LDST : WR;
6354 %}
6355
6356 // Load - reg, mem
6357 // Eg. LDR x0, <mem>
6358 pipe_class iload_reg_mem(iRegI dst, memory mem)
6359 %{
6360 single_instruction;
6361 dst : WR(write);
6362 mem : ISS(read);
6363 INS01 : ISS;
6364 LDST : WR;
6365 %}
6366
6367 // Load - reg, reg
6368 // Eg. LDR x0, [sp, x1]
6369 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6370 %{
6371 single_instruction;
6372 dst : WR(write);
6373 src : ISS(read);
6374 INS01 : ISS;
6375 LDST : WR;
6376 %}
6377
6378 //------- Store pipeline operations -----------------------
6379
6380 // Store - zr, mem
6381 // Eg. STR zr, <mem>
6382 pipe_class istore_mem(memory mem)
6383 %{
6384 single_instruction;
6385 mem : ISS(read);
6386 INS01 : ISS;
6387 LDST : WR;
6388 %}
6389
6390 // Store - reg, mem
6391 // Eg. STR x0, <mem>
6392 pipe_class istore_reg_mem(iRegI src, memory mem)
6393 %{
6394 single_instruction;
6395 mem : ISS(read);
6396 src : EX2(read);
6397 INS01 : ISS;
6398 LDST : WR;
6399 %}
6400
6401 // Store - reg, reg
6402 // Eg. STR x0, [sp, x1]
6403 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6404 %{
6405 single_instruction;
6406 dst : ISS(read);
6407 src : EX2(read);
6408 INS01 : ISS;
6409 LDST : WR;
6410 %}
6411
6412 //------- Store pipeline operations -----------------------
6413
6414 // Branch
6415 pipe_class pipe_branch()
6416 %{
6417 single_instruction;
6418 INS01 : ISS;
6419 BRANCH : EX1;
6420 %}
6421
6422 // Conditional branch
6423 pipe_class pipe_branch_cond(rFlagsReg cr)
6424 %{
6425 single_instruction;
6426 cr : EX1(read);
6427 INS01 : ISS;
6428 BRANCH : EX1;
6429 %}
6430
6431 // Compare & Branch
6432 // EG. CBZ/CBNZ
6433 pipe_class pipe_cmp_branch(iRegI op1)
6434 %{
6435 single_instruction;
6436 op1 : EX1(read);
6437 INS01 : ISS;
6438 BRANCH : EX1;
6439 %}
6440
6441 //------- Synchronisation operations ----------------------
6442
6443 // Any operation requiring serialization.
6444 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6445 pipe_class pipe_serial()
6446 %{
6447 single_instruction;
6448 force_serialization;
6449 fixed_latency(16);
6450 INS01 : ISS(2); // Cannot dual issue with any other instruction
6451 LDST : WR;
6452 %}
6453
6454 // Generic big/slow expanded idiom - also serialized
6455 pipe_class pipe_slow()
6456 %{
6457 instruction_count(10);
6458 multiple_bundles;
6459 force_serialization;
6460 fixed_latency(16);
6461 INS01 : ISS(2); // Cannot dual issue with any other instruction
6462 LDST : WR;
6463 %}
6464
6465 // Empty pipeline class
6466 pipe_class pipe_class_empty()
6467 %{
6468 single_instruction;
6469 fixed_latency(0);
6470 %}
6471
6472 // Default pipeline class.
6473 pipe_class pipe_class_default()
6474 %{
6475 single_instruction;
6476 fixed_latency(2);
6477 %}
6478
6479 // Pipeline class for compares.
6480 pipe_class pipe_class_compare()
6481 %{
6482 single_instruction;
6483 fixed_latency(16);
6484 %}
6485
6486 // Pipeline class for memory operations.
6487 pipe_class pipe_class_memory()
6488 %{
6489 single_instruction;
6490 fixed_latency(16);
6491 %}
6492
6493 // Pipeline class for call.
6494 pipe_class pipe_class_call()
6495 %{
6496 single_instruction;
6497 fixed_latency(100);
6498 %}
6499
6500 // Define the class for the Nop node.
6501 define %{
6502 MachNop = pipe_class_empty;
6503 %}
6504
6505 %}
6506 //----------INSTRUCTIONS-------------------------------------------------------
6507 //
6508 // match -- States which machine-independent subtree may be replaced
6509 // by this instruction.
6510 // ins_cost -- The estimated cost of this instruction is used by instruction
6511 // selection to identify a minimum cost tree of machine
6512 // instructions that matches a tree of machine-independent
6513 // instructions.
6514 // format -- A string providing the disassembly for this instruction.
6515 // The value of an instruction's operand may be inserted
6516 // by referring to it with a '$' prefix.
6517 // opcode -- Three instruction opcodes may be provided. These are referred
6518 // to within an encode class as $primary, $secondary, and $tertiary
6519 // rrspectively. The primary opcode is commonly used to
6520 // indicate the type of machine instruction, while secondary
6521 // and tertiary are often used for prefix options or addressing
6522 // modes.
6523 // ins_encode -- A list of encode classes with parameters. The encode class
6524 // name must have been defined in an 'enc_class' specification
6525 // in the encode section of the architecture description.
6526
6527 // ============================================================================
6528 // Memory (Load/Store) Instructions
6529
6530 // Load Instructions
6531
6532 // Load Byte (8 bit signed)
6533 instruct loadB(iRegINoSp dst, memory1 mem)
6534 %{
6535 match(Set dst (LoadB mem));
6536 predicate(!needs_acquiring_load(n));
6537
6538 ins_cost(4 * INSN_COST);
6539 format %{ "ldrsbw $dst, $mem\t# byte" %}
6540
6541 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6542
6543 ins_pipe(iload_reg_mem);
6544 %}
6545
6546 // Load Byte (8 bit signed) into long
6547 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6548 %{
6549 match(Set dst (ConvI2L (LoadB mem)));
6550 predicate(!needs_acquiring_load(n->in(1)));
6551
6552 ins_cost(4 * INSN_COST);
6553 format %{ "ldrsb $dst, $mem\t# byte" %}
6554
6555 ins_encode(aarch64_enc_ldrsb(dst, mem));
6556
6557 ins_pipe(iload_reg_mem);
6558 %}
6559
6560 // Load Byte (8 bit unsigned)
6561 instruct loadUB(iRegINoSp dst, memory1 mem)
6562 %{
6563 match(Set dst (LoadUB mem));
6564 predicate(!needs_acquiring_load(n));
6565
6566 ins_cost(4 * INSN_COST);
6567 format %{ "ldrbw $dst, $mem\t# byte" %}
6568
6569 ins_encode(aarch64_enc_ldrb(dst, mem));
6570
6571 ins_pipe(iload_reg_mem);
6572 %}
6573
6574 // Load Byte (8 bit unsigned) into long
6575 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6576 %{
6577 match(Set dst (ConvI2L (LoadUB mem)));
6578 predicate(!needs_acquiring_load(n->in(1)));
6579
6580 ins_cost(4 * INSN_COST);
6581 format %{ "ldrb $dst, $mem\t# byte" %}
6582
6583 ins_encode(aarch64_enc_ldrb(dst, mem));
6584
6585 ins_pipe(iload_reg_mem);
6586 %}
6587
6588 // Load Short (16 bit signed)
6589 instruct loadS(iRegINoSp dst, memory2 mem)
6590 %{
6591 match(Set dst (LoadS mem));
6592 predicate(!needs_acquiring_load(n));
6593
6594 ins_cost(4 * INSN_COST);
6595 format %{ "ldrshw $dst, $mem\t# short" %}
6596
6597 ins_encode(aarch64_enc_ldrshw(dst, mem));
6598
6599 ins_pipe(iload_reg_mem);
6600 %}
6601
6602 // Load Short (16 bit signed) into long
6603 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6604 %{
6605 match(Set dst (ConvI2L (LoadS mem)));
6606 predicate(!needs_acquiring_load(n->in(1)));
6607
6608 ins_cost(4 * INSN_COST);
6609 format %{ "ldrsh $dst, $mem\t# short" %}
6610
6611 ins_encode(aarch64_enc_ldrsh(dst, mem));
6612
6613 ins_pipe(iload_reg_mem);
6614 %}
6615
6616 // Load Char (16 bit unsigned)
6617 instruct loadUS(iRegINoSp dst, memory2 mem)
6618 %{
6619 match(Set dst (LoadUS mem));
6620 predicate(!needs_acquiring_load(n));
6621
6622 ins_cost(4 * INSN_COST);
6623 format %{ "ldrh $dst, $mem\t# short" %}
6624
6625 ins_encode(aarch64_enc_ldrh(dst, mem));
6626
6627 ins_pipe(iload_reg_mem);
6628 %}
6629
6630 // Load Short/Char (16 bit unsigned) into long
6631 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6632 %{
6633 match(Set dst (ConvI2L (LoadUS mem)));
6634 predicate(!needs_acquiring_load(n->in(1)));
6635
6636 ins_cost(4 * INSN_COST);
6637 format %{ "ldrh $dst, $mem\t# short" %}
6638
6639 ins_encode(aarch64_enc_ldrh(dst, mem));
6640
6641 ins_pipe(iload_reg_mem);
6642 %}
6643
6644 // Load Integer (32 bit signed)
6645 instruct loadI(iRegINoSp dst, memory4 mem)
6646 %{
6647 match(Set dst (LoadI mem));
6648 predicate(!needs_acquiring_load(n));
6649
6650 ins_cost(4 * INSN_COST);
6651 format %{ "ldrw $dst, $mem\t# int" %}
6652
6653 ins_encode(aarch64_enc_ldrw(dst, mem));
6654
6655 ins_pipe(iload_reg_mem);
6656 %}
6657
6658 // Load Integer (32 bit signed) into long
6659 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6660 %{
6661 match(Set dst (ConvI2L (LoadI mem)));
6662 predicate(!needs_acquiring_load(n->in(1)));
6663
6664 ins_cost(4 * INSN_COST);
6665 format %{ "ldrsw $dst, $mem\t# int" %}
6666
6667 ins_encode(aarch64_enc_ldrsw(dst, mem));
6668
6669 ins_pipe(iload_reg_mem);
6670 %}
6671
6672 // Load Integer (32 bit unsigned) into long
6673 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6674 %{
6675 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6676 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6677
6678 ins_cost(4 * INSN_COST);
6679 format %{ "ldrw $dst, $mem\t# int" %}
6680
6681 ins_encode(aarch64_enc_ldrw(dst, mem));
6682
6683 ins_pipe(iload_reg_mem);
6684 %}
6685
6686 // Load Long (64 bit signed)
6687 instruct loadL(iRegLNoSp dst, memory8 mem)
6688 %{
6689 match(Set dst (LoadL mem));
6690 predicate(!needs_acquiring_load(n));
6691
6692 ins_cost(4 * INSN_COST);
6693 format %{ "ldr $dst, $mem\t# int" %}
6694
6695 ins_encode(aarch64_enc_ldr(dst, mem));
6696
6697 ins_pipe(iload_reg_mem);
6698 %}
6699
6700 // Load Range
6701 instruct loadRange(iRegINoSp dst, memory4 mem)
6702 %{
6703 match(Set dst (LoadRange mem));
6704
6705 ins_cost(4 * INSN_COST);
6706 format %{ "ldrw $dst, $mem\t# range" %}
6707
6708 ins_encode(aarch64_enc_ldrw(dst, mem));
6709
6710 ins_pipe(iload_reg_mem);
6711 %}
6712
6713 // Load Pointer
6714 instruct loadP(iRegPNoSp dst, memory8 mem)
6715 %{
6716 match(Set dst (LoadP mem));
6717 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6718
6719 ins_cost(4 * INSN_COST);
6720 format %{ "ldr $dst, $mem\t# ptr" %}
6721
6722 ins_encode(aarch64_enc_ldr(dst, mem));
6723
6724 ins_pipe(iload_reg_mem);
6725 %}
6726
6727 // Load Compressed Pointer
6728 instruct loadN(iRegNNoSp dst, memory4 mem)
6729 %{
6730 match(Set dst (LoadN mem));
6731 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6732
6733 ins_cost(4 * INSN_COST);
6734 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6735
6736 ins_encode(aarch64_enc_ldrw(dst, mem));
6737
6738 ins_pipe(iload_reg_mem);
6739 %}
6740
6741 // Load Klass Pointer
6742 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6743 %{
6744 match(Set dst (LoadKlass mem));
6745 predicate(!needs_acquiring_load(n));
6746
6747 ins_cost(4 * INSN_COST);
6748 format %{ "ldr $dst, $mem\t# class" %}
6749
6750 ins_encode(aarch64_enc_ldr(dst, mem));
6751
6752 ins_pipe(iload_reg_mem);
6753 %}
6754
6755 // Load Narrow Klass Pointer
6756 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6757 %{
6758 match(Set dst (LoadNKlass mem));
6759 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6760
6761 ins_cost(4 * INSN_COST);
6762 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6763
6764 ins_encode(aarch64_enc_ldrw(dst, mem));
6765
6766 ins_pipe(iload_reg_mem);
6767 %}
6768
6769 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6770 %{
6771 match(Set dst (LoadNKlass mem));
6772 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6773
6774 ins_cost(4 * INSN_COST);
6775 format %{
6776 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6777 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6778 %}
6779 ins_encode %{
6780 // inlined aarch64_enc_ldrw
6781 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6782 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6783 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6784 %}
6785 ins_pipe(iload_reg_mem);
6786 %}
6787
6788 // Load Float
6789 instruct loadF(vRegF dst, memory4 mem)
6790 %{
6791 match(Set dst (LoadF mem));
6792 predicate(!needs_acquiring_load(n));
6793
6794 ins_cost(4 * INSN_COST);
6795 format %{ "ldrs $dst, $mem\t# float" %}
6796
6797 ins_encode( aarch64_enc_ldrs(dst, mem) );
6798
6799 ins_pipe(pipe_class_memory);
6800 %}
6801
6802 // Load Double
6803 instruct loadD(vRegD dst, memory8 mem)
6804 %{
6805 match(Set dst (LoadD mem));
6806 predicate(!needs_acquiring_load(n));
6807
6808 ins_cost(4 * INSN_COST);
6809 format %{ "ldrd $dst, $mem\t# double" %}
6810
6811 ins_encode( aarch64_enc_ldrd(dst, mem) );
6812
6813 ins_pipe(pipe_class_memory);
6814 %}
6815
6816
6817 // Load Int Constant
6818 instruct loadConI(iRegINoSp dst, immI src)
6819 %{
6820 match(Set dst src);
6821
6822 ins_cost(INSN_COST);
6823 format %{ "mov $dst, $src\t# int" %}
6824
6825 ins_encode( aarch64_enc_movw_imm(dst, src) );
6826
6827 ins_pipe(ialu_imm);
6828 %}
6829
6830 // Load Long Constant
6831 instruct loadConL(iRegLNoSp dst, immL src)
6832 %{
6833 match(Set dst src);
6834
6835 ins_cost(INSN_COST);
6836 format %{ "mov $dst, $src\t# long" %}
6837
6838 ins_encode( aarch64_enc_mov_imm(dst, src) );
6839
6840 ins_pipe(ialu_imm);
6841 %}
6842
6843 // Load Pointer Constant
6844
6845 instruct loadConP(iRegPNoSp dst, immP con)
6846 %{
6847 match(Set dst con);
6848
6849 ins_cost(INSN_COST * 4);
6850 format %{
6851 "mov $dst, $con\t# ptr\n\t"
6852 %}
6853
6854 ins_encode(aarch64_enc_mov_p(dst, con));
6855
6856 ins_pipe(ialu_imm);
6857 %}
6858
6859 // Load Null Pointer Constant
6860
6861 instruct loadConP0(iRegPNoSp dst, immP0 con)
6862 %{
6863 match(Set dst con);
6864
6865 ins_cost(INSN_COST);
6866 format %{ "mov $dst, $con\t# nullptr ptr" %}
6867
6868 ins_encode(aarch64_enc_mov_p0(dst, con));
6869
6870 ins_pipe(ialu_imm);
6871 %}
6872
6873 // Load Pointer Constant One
6874
6875 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6876 %{
6877 match(Set dst con);
6878
6879 ins_cost(INSN_COST);
6880 format %{ "mov $dst, $con\t# nullptr ptr" %}
6881
6882 ins_encode(aarch64_enc_mov_p1(dst, con));
6883
6884 ins_pipe(ialu_imm);
6885 %}
6886
6887 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6888 %{
6889 match(Set dst con);
6890
6891 ins_cost(INSN_COST);
6892 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6893
6894 ins_encode %{
6895 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6896 %}
6897
6898 ins_pipe(ialu_imm);
6899 %}
6900
6901 // Load Narrow Pointer Constant
6902
6903 instruct loadConN(iRegNNoSp dst, immN con)
6904 %{
6905 match(Set dst con);
6906
6907 ins_cost(INSN_COST * 4);
6908 format %{ "mov $dst, $con\t# compressed ptr" %}
6909
6910 ins_encode(aarch64_enc_mov_n(dst, con));
6911
6912 ins_pipe(ialu_imm);
6913 %}
6914
6915 // Load Narrow Null Pointer Constant
6916
6917 instruct loadConN0(iRegNNoSp dst, immN0 con)
6918 %{
6919 match(Set dst con);
6920
6921 ins_cost(INSN_COST);
6922 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6923
6924 ins_encode(aarch64_enc_mov_n0(dst, con));
6925
6926 ins_pipe(ialu_imm);
6927 %}
6928
6929 // Load Narrow Klass Constant
6930
6931 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6932 %{
6933 match(Set dst con);
6934
6935 ins_cost(INSN_COST);
6936 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6937
6938 ins_encode(aarch64_enc_mov_nk(dst, con));
6939
6940 ins_pipe(ialu_imm);
6941 %}
6942
6943 // Load Packed Float Constant
6944
6945 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6946 match(Set dst con);
6947 ins_cost(INSN_COST * 4);
6948 format %{ "fmovs $dst, $con"%}
6949 ins_encode %{
6950 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6951 %}
6952
6953 ins_pipe(fp_imm_s);
6954 %}
6955
6956 // Load Float Constant
6957
6958 instruct loadConF(vRegF dst, immF con) %{
6959 match(Set dst con);
6960
6961 ins_cost(INSN_COST * 4);
6962
6963 format %{
6964 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6965 %}
6966
6967 ins_encode %{
6968 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6969 %}
6970
6971 ins_pipe(fp_load_constant_s);
6972 %}
6973
6974 // Load Packed Double Constant
6975
6976 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6977 match(Set dst con);
6978 ins_cost(INSN_COST);
6979 format %{ "fmovd $dst, $con"%}
6980 ins_encode %{
6981 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6982 %}
6983
6984 ins_pipe(fp_imm_d);
6985 %}
6986
6987 // Load Double Constant
6988
6989 instruct loadConD(vRegD dst, immD con) %{
6990 match(Set dst con);
6991
6992 ins_cost(INSN_COST * 5);
6993 format %{
6994 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6995 %}
6996
6997 ins_encode %{
6998 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6999 %}
7000
7001 ins_pipe(fp_load_constant_d);
7002 %}
7003
7004 // Load Half Float Constant
7005 instruct loadConH(vRegF dst, immH con) %{
7006 match(Set dst con);
7007 format %{ "mov rscratch1, $con\n\t"
7008 "fmov $dst, rscratch1"
7009 %}
7010 ins_encode %{
7011 __ movw(rscratch1, (uint32_t)$con$$constant);
7012 __ fmovs($dst$$FloatRegister, rscratch1);
7013 %}
7014 ins_pipe(pipe_class_default);
7015 %}
7016
7017 // Store Instructions
7018
7019 // Store Byte
7020 instruct storeB(iRegIorL2I src, memory1 mem)
7021 %{
7022 match(Set mem (StoreB mem src));
7023 predicate(!needs_releasing_store(n));
7024
7025 ins_cost(INSN_COST);
7026 format %{ "strb $src, $mem\t# byte" %}
7027
7028 ins_encode(aarch64_enc_strb(src, mem));
7029
7030 ins_pipe(istore_reg_mem);
7031 %}
7032
7033
7034 instruct storeimmB0(immI0 zero, memory1 mem)
7035 %{
7036 match(Set mem (StoreB mem zero));
7037 predicate(!needs_releasing_store(n));
7038
7039 ins_cost(INSN_COST);
7040 format %{ "strb rscractch2, $mem\t# byte" %}
7041
7042 ins_encode(aarch64_enc_strb0(mem));
7043
7044 ins_pipe(istore_mem);
7045 %}
7046
7047 // Store Char/Short
7048 instruct storeC(iRegIorL2I src, memory2 mem)
7049 %{
7050 match(Set mem (StoreC mem src));
7051 predicate(!needs_releasing_store(n));
7052
7053 ins_cost(INSN_COST);
7054 format %{ "strh $src, $mem\t# short" %}
7055
7056 ins_encode(aarch64_enc_strh(src, mem));
7057
7058 ins_pipe(istore_reg_mem);
7059 %}
7060
7061 instruct storeimmC0(immI0 zero, memory2 mem)
7062 %{
7063 match(Set mem (StoreC mem zero));
7064 predicate(!needs_releasing_store(n));
7065
7066 ins_cost(INSN_COST);
7067 format %{ "strh zr, $mem\t# short" %}
7068
7069 ins_encode(aarch64_enc_strh0(mem));
7070
7071 ins_pipe(istore_mem);
7072 %}
7073
7074 // Store Integer
7075
7076 instruct storeI(iRegIorL2I src, memory4 mem)
7077 %{
7078 match(Set mem(StoreI mem src));
7079 predicate(!needs_releasing_store(n));
7080
7081 ins_cost(INSN_COST);
7082 format %{ "strw $src, $mem\t# int" %}
7083
7084 ins_encode(aarch64_enc_strw(src, mem));
7085
7086 ins_pipe(istore_reg_mem);
7087 %}
7088
7089 instruct storeimmI0(immI0 zero, memory4 mem)
7090 %{
7091 match(Set mem(StoreI mem zero));
7092 predicate(!needs_releasing_store(n));
7093
7094 ins_cost(INSN_COST);
7095 format %{ "strw zr, $mem\t# int" %}
7096
7097 ins_encode(aarch64_enc_strw0(mem));
7098
7099 ins_pipe(istore_mem);
7100 %}
7101
7102 // Store Long (64 bit signed)
7103 instruct storeL(iRegL src, memory8 mem)
7104 %{
7105 match(Set mem (StoreL mem src));
7106 predicate(!needs_releasing_store(n));
7107
7108 ins_cost(INSN_COST);
7109 format %{ "str $src, $mem\t# int" %}
7110
7111 ins_encode(aarch64_enc_str(src, mem));
7112
7113 ins_pipe(istore_reg_mem);
7114 %}
7115
7116 // Store Long (64 bit signed)
7117 instruct storeimmL0(immL0 zero, memory8 mem)
7118 %{
7119 match(Set mem (StoreL mem zero));
7120 predicate(!needs_releasing_store(n));
7121
7122 ins_cost(INSN_COST);
7123 format %{ "str zr, $mem\t# int" %}
7124
7125 ins_encode(aarch64_enc_str0(mem));
7126
7127 ins_pipe(istore_mem);
7128 %}
7129
7130 // Store Pointer
7131 instruct storeP(iRegP src, memory8 mem)
7132 %{
7133 match(Set mem (StoreP mem src));
7134 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7135
7136 ins_cost(INSN_COST);
7137 format %{ "str $src, $mem\t# ptr" %}
7138
7139 ins_encode(aarch64_enc_str(src, mem));
7140
7141 ins_pipe(istore_reg_mem);
7142 %}
7143
7144 // Store Pointer
7145 instruct storeimmP0(immP0 zero, memory8 mem)
7146 %{
7147 match(Set mem (StoreP mem zero));
7148 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7149
7150 ins_cost(INSN_COST);
7151 format %{ "str zr, $mem\t# ptr" %}
7152
7153 ins_encode(aarch64_enc_str0(mem));
7154
7155 ins_pipe(istore_mem);
7156 %}
7157
7158 // Store Compressed Pointer
7159 instruct storeN(iRegN src, memory4 mem)
7160 %{
7161 match(Set mem (StoreN mem src));
7162 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7163
7164 ins_cost(INSN_COST);
7165 format %{ "strw $src, $mem\t# compressed ptr" %}
7166
7167 ins_encode(aarch64_enc_strw(src, mem));
7168
7169 ins_pipe(istore_reg_mem);
7170 %}
7171
7172 instruct storeImmN0(immN0 zero, memory4 mem)
7173 %{
7174 match(Set mem (StoreN mem zero));
7175 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7176
7177 ins_cost(INSN_COST);
7178 format %{ "strw zr, $mem\t# compressed ptr" %}
7179
7180 ins_encode(aarch64_enc_strw0(mem));
7181
7182 ins_pipe(istore_mem);
7183 %}
7184
7185 // Store Float
7186 instruct storeF(vRegF src, memory4 mem)
7187 %{
7188 match(Set mem (StoreF mem src));
7189 predicate(!needs_releasing_store(n));
7190
7191 ins_cost(INSN_COST);
7192 format %{ "strs $src, $mem\t# float" %}
7193
7194 ins_encode( aarch64_enc_strs(src, mem) );
7195
7196 ins_pipe(pipe_class_memory);
7197 %}
7198
7199 // TODO
7200 // implement storeImmF0 and storeFImmPacked
7201
7202 // Store Double
7203 instruct storeD(vRegD src, memory8 mem)
7204 %{
7205 match(Set mem (StoreD mem src));
7206 predicate(!needs_releasing_store(n));
7207
7208 ins_cost(INSN_COST);
7209 format %{ "strd $src, $mem\t# double" %}
7210
7211 ins_encode( aarch64_enc_strd(src, mem) );
7212
7213 ins_pipe(pipe_class_memory);
7214 %}
7215
7216 // Store Compressed Klass Pointer
7217 instruct storeNKlass(iRegN src, memory4 mem)
7218 %{
7219 predicate(!needs_releasing_store(n));
7220 match(Set mem (StoreNKlass mem src));
7221
7222 ins_cost(INSN_COST);
7223 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7224
7225 ins_encode(aarch64_enc_strw(src, mem));
7226
7227 ins_pipe(istore_reg_mem);
7228 %}
7229
7230 // TODO
7231 // implement storeImmD0 and storeDImmPacked
7232
7233 // prefetch instructions
7234 // Must be safe to execute with invalid address (cannot fault).
7235
7236 instruct prefetchalloc( memory8 mem ) %{
7237 match(PrefetchAllocation mem);
7238
7239 ins_cost(INSN_COST);
7240 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7241
7242 ins_encode( aarch64_enc_prefetchw(mem) );
7243
7244 ins_pipe(iload_prefetch);
7245 %}
7246
7247 // ---------------- volatile loads and stores ----------------
7248
7249 // Load Byte (8 bit signed)
7250 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7251 %{
7252 match(Set dst (LoadB mem));
7253
7254 ins_cost(VOLATILE_REF_COST);
7255 format %{ "ldarsb $dst, $mem\t# byte" %}
7256
7257 ins_encode(aarch64_enc_ldarsb(dst, mem));
7258
7259 ins_pipe(pipe_serial);
7260 %}
7261
7262 // Load Byte (8 bit signed) into long
7263 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7264 %{
7265 match(Set dst (ConvI2L (LoadB mem)));
7266
7267 ins_cost(VOLATILE_REF_COST);
7268 format %{ "ldarsb $dst, $mem\t# byte" %}
7269
7270 ins_encode(aarch64_enc_ldarsb(dst, mem));
7271
7272 ins_pipe(pipe_serial);
7273 %}
7274
7275 // Load Byte (8 bit unsigned)
7276 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7277 %{
7278 match(Set dst (LoadUB mem));
7279
7280 ins_cost(VOLATILE_REF_COST);
7281 format %{ "ldarb $dst, $mem\t# byte" %}
7282
7283 ins_encode(aarch64_enc_ldarb(dst, mem));
7284
7285 ins_pipe(pipe_serial);
7286 %}
7287
7288 // Load Byte (8 bit unsigned) into long
7289 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7290 %{
7291 match(Set dst (ConvI2L (LoadUB mem)));
7292
7293 ins_cost(VOLATILE_REF_COST);
7294 format %{ "ldarb $dst, $mem\t# byte" %}
7295
7296 ins_encode(aarch64_enc_ldarb(dst, mem));
7297
7298 ins_pipe(pipe_serial);
7299 %}
7300
7301 // Load Short (16 bit signed)
7302 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7303 %{
7304 match(Set dst (LoadS mem));
7305
7306 ins_cost(VOLATILE_REF_COST);
7307 format %{ "ldarshw $dst, $mem\t# short" %}
7308
7309 ins_encode(aarch64_enc_ldarshw(dst, mem));
7310
7311 ins_pipe(pipe_serial);
7312 %}
7313
7314 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7315 %{
7316 match(Set dst (LoadUS mem));
7317
7318 ins_cost(VOLATILE_REF_COST);
7319 format %{ "ldarhw $dst, $mem\t# short" %}
7320
7321 ins_encode(aarch64_enc_ldarhw(dst, mem));
7322
7323 ins_pipe(pipe_serial);
7324 %}
7325
7326 // Load Short/Char (16 bit unsigned) into long
7327 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7328 %{
7329 match(Set dst (ConvI2L (LoadUS mem)));
7330
7331 ins_cost(VOLATILE_REF_COST);
7332 format %{ "ldarh $dst, $mem\t# short" %}
7333
7334 ins_encode(aarch64_enc_ldarh(dst, mem));
7335
7336 ins_pipe(pipe_serial);
7337 %}
7338
7339 // Load Short/Char (16 bit signed) into long
7340 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7341 %{
7342 match(Set dst (ConvI2L (LoadS mem)));
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "ldarh $dst, $mem\t# short" %}
7346
7347 ins_encode(aarch64_enc_ldarsh(dst, mem));
7348
7349 ins_pipe(pipe_serial);
7350 %}
7351
7352 // Load Integer (32 bit signed)
7353 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (LoadI mem));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldarw $dst, $mem\t# int" %}
7359
7360 ins_encode(aarch64_enc_ldarw(dst, mem));
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Integer (32 bit unsigned) into long
7366 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7367 %{
7368 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldarw $dst, $mem\t# int" %}
7372
7373 ins_encode(aarch64_enc_ldarw(dst, mem));
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 // Load Long (64 bit signed)
7379 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7380 %{
7381 match(Set dst (LoadL mem));
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "ldar $dst, $mem\t# int" %}
7385
7386 ins_encode(aarch64_enc_ldar(dst, mem));
7387
7388 ins_pipe(pipe_serial);
7389 %}
7390
7391 // Load Pointer
7392 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7393 %{
7394 match(Set dst (LoadP mem));
7395 predicate(n->as_Load()->barrier_data() == 0);
7396
7397 ins_cost(VOLATILE_REF_COST);
7398 format %{ "ldar $dst, $mem\t# ptr" %}
7399
7400 ins_encode(aarch64_enc_ldar(dst, mem));
7401
7402 ins_pipe(pipe_serial);
7403 %}
7404
7405 // Load Compressed Pointer
7406 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7407 %{
7408 match(Set dst (LoadN mem));
7409 predicate(n->as_Load()->barrier_data() == 0);
7410
7411 ins_cost(VOLATILE_REF_COST);
7412 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7413
7414 ins_encode(aarch64_enc_ldarw(dst, mem));
7415
7416 ins_pipe(pipe_serial);
7417 %}
7418
7419 // Load Float
7420 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7421 %{
7422 match(Set dst (LoadF mem));
7423
7424 ins_cost(VOLATILE_REF_COST);
7425 format %{ "ldars $dst, $mem\t# float" %}
7426
7427 ins_encode( aarch64_enc_fldars(dst, mem) );
7428
7429 ins_pipe(pipe_serial);
7430 %}
7431
7432 // Load Double
7433 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7434 %{
7435 match(Set dst (LoadD mem));
7436
7437 ins_cost(VOLATILE_REF_COST);
7438 format %{ "ldard $dst, $mem\t# double" %}
7439
7440 ins_encode( aarch64_enc_fldard(dst, mem) );
7441
7442 ins_pipe(pipe_serial);
7443 %}
7444
7445 // Store Byte
7446 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7447 %{
7448 match(Set mem (StoreB mem src));
7449
7450 ins_cost(VOLATILE_REF_COST);
7451 format %{ "stlrb $src, $mem\t# byte" %}
7452
7453 ins_encode(aarch64_enc_stlrb(src, mem));
7454
7455 ins_pipe(pipe_class_memory);
7456 %}
7457
7458 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7459 %{
7460 match(Set mem (StoreB mem zero));
7461
7462 ins_cost(VOLATILE_REF_COST);
7463 format %{ "stlrb zr, $mem\t# byte" %}
7464
7465 ins_encode(aarch64_enc_stlrb0(mem));
7466
7467 ins_pipe(pipe_class_memory);
7468 %}
7469
7470 // Store Char/Short
7471 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7472 %{
7473 match(Set mem (StoreC mem src));
7474
7475 ins_cost(VOLATILE_REF_COST);
7476 format %{ "stlrh $src, $mem\t# short" %}
7477
7478 ins_encode(aarch64_enc_stlrh(src, mem));
7479
7480 ins_pipe(pipe_class_memory);
7481 %}
7482
7483 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7484 %{
7485 match(Set mem (StoreC mem zero));
7486
7487 ins_cost(VOLATILE_REF_COST);
7488 format %{ "stlrh zr, $mem\t# short" %}
7489
7490 ins_encode(aarch64_enc_stlrh0(mem));
7491
7492 ins_pipe(pipe_class_memory);
7493 %}
7494
7495 // Store Integer
7496
7497 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7498 %{
7499 match(Set mem(StoreI mem src));
7500
7501 ins_cost(VOLATILE_REF_COST);
7502 format %{ "stlrw $src, $mem\t# int" %}
7503
7504 ins_encode(aarch64_enc_stlrw(src, mem));
7505
7506 ins_pipe(pipe_class_memory);
7507 %}
7508
7509 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7510 %{
7511 match(Set mem(StoreI mem zero));
7512
7513 ins_cost(VOLATILE_REF_COST);
7514 format %{ "stlrw zr, $mem\t# int" %}
7515
7516 ins_encode(aarch64_enc_stlrw0(mem));
7517
7518 ins_pipe(pipe_class_memory);
7519 %}
7520
7521 // Store Long (64 bit signed)
7522 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7523 %{
7524 match(Set mem (StoreL mem src));
7525
7526 ins_cost(VOLATILE_REF_COST);
7527 format %{ "stlr $src, $mem\t# int" %}
7528
7529 ins_encode(aarch64_enc_stlr(src, mem));
7530
7531 ins_pipe(pipe_class_memory);
7532 %}
7533
7534 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7535 %{
7536 match(Set mem (StoreL mem zero));
7537
7538 ins_cost(VOLATILE_REF_COST);
7539 format %{ "stlr zr, $mem\t# int" %}
7540
7541 ins_encode(aarch64_enc_stlr0(mem));
7542
7543 ins_pipe(pipe_class_memory);
7544 %}
7545
7546 // Store Pointer
7547 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7548 %{
7549 match(Set mem (StoreP mem src));
7550 predicate(n->as_Store()->barrier_data() == 0);
7551
7552 ins_cost(VOLATILE_REF_COST);
7553 format %{ "stlr $src, $mem\t# ptr" %}
7554
7555 ins_encode(aarch64_enc_stlr(src, mem));
7556
7557 ins_pipe(pipe_class_memory);
7558 %}
7559
7560 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7561 %{
7562 match(Set mem (StoreP mem zero));
7563 predicate(n->as_Store()->barrier_data() == 0);
7564
7565 ins_cost(VOLATILE_REF_COST);
7566 format %{ "stlr zr, $mem\t# ptr" %}
7567
7568 ins_encode(aarch64_enc_stlr0(mem));
7569
7570 ins_pipe(pipe_class_memory);
7571 %}
7572
7573 // Store Compressed Pointer
7574 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7575 %{
7576 match(Set mem (StoreN mem src));
7577 predicate(n->as_Store()->barrier_data() == 0);
7578
7579 ins_cost(VOLATILE_REF_COST);
7580 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7581
7582 ins_encode(aarch64_enc_stlrw(src, mem));
7583
7584 ins_pipe(pipe_class_memory);
7585 %}
7586
7587 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7588 %{
7589 match(Set mem (StoreN mem zero));
7590 predicate(n->as_Store()->barrier_data() == 0);
7591
7592 ins_cost(VOLATILE_REF_COST);
7593 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7594
7595 ins_encode(aarch64_enc_stlrw0(mem));
7596
7597 ins_pipe(pipe_class_memory);
7598 %}
7599
7600 // Store Float
7601 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7602 %{
7603 match(Set mem (StoreF mem src));
7604
7605 ins_cost(VOLATILE_REF_COST);
7606 format %{ "stlrs $src, $mem\t# float" %}
7607
7608 ins_encode( aarch64_enc_fstlrs(src, mem) );
7609
7610 ins_pipe(pipe_class_memory);
7611 %}
7612
7613 // TODO
7614 // implement storeImmF0 and storeFImmPacked
7615
7616 // Store Double
7617 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7618 %{
7619 match(Set mem (StoreD mem src));
7620
7621 ins_cost(VOLATILE_REF_COST);
7622 format %{ "stlrd $src, $mem\t# double" %}
7623
7624 ins_encode( aarch64_enc_fstlrd(src, mem) );
7625
7626 ins_pipe(pipe_class_memory);
7627 %}
7628
7629 // ---------------- end of volatile loads and stores ----------------
7630
7631 instruct cacheWB(indirect addr)
7632 %{
7633 predicate(VM_Version::supports_data_cache_line_flush());
7634 match(CacheWB addr);
7635
7636 ins_cost(100);
7637 format %{"cache wb $addr" %}
7638 ins_encode %{
7639 assert($addr->index_position() < 0, "should be");
7640 assert($addr$$disp == 0, "should be");
7641 __ cache_wb(Address($addr$$base$$Register, 0));
7642 %}
7643 ins_pipe(pipe_slow); // XXX
7644 %}
7645
7646 instruct cacheWBPreSync()
7647 %{
7648 predicate(VM_Version::supports_data_cache_line_flush());
7649 match(CacheWBPreSync);
7650
7651 ins_cost(100);
7652 format %{"cache wb presync" %}
7653 ins_encode %{
7654 __ cache_wbsync(true);
7655 %}
7656 ins_pipe(pipe_slow); // XXX
7657 %}
7658
7659 instruct cacheWBPostSync()
7660 %{
7661 predicate(VM_Version::supports_data_cache_line_flush());
7662 match(CacheWBPostSync);
7663
7664 ins_cost(100);
7665 format %{"cache wb postsync" %}
7666 ins_encode %{
7667 __ cache_wbsync(false);
7668 %}
7669 ins_pipe(pipe_slow); // XXX
7670 %}
7671
7672 // ============================================================================
7673 // BSWAP Instructions
7674
7675 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7676 match(Set dst (ReverseBytesI src));
7677
7678 ins_cost(INSN_COST);
7679 format %{ "revw $dst, $src" %}
7680
7681 ins_encode %{
7682 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7683 %}
7684
7685 ins_pipe(ialu_reg);
7686 %}
7687
7688 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7689 match(Set dst (ReverseBytesL src));
7690
7691 ins_cost(INSN_COST);
7692 format %{ "rev $dst, $src" %}
7693
7694 ins_encode %{
7695 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7696 %}
7697
7698 ins_pipe(ialu_reg);
7699 %}
7700
7701 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7702 match(Set dst (ReverseBytesUS src));
7703
7704 ins_cost(INSN_COST);
7705 format %{ "rev16w $dst, $src" %}
7706
7707 ins_encode %{
7708 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7709 %}
7710
7711 ins_pipe(ialu_reg);
7712 %}
7713
7714 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7715 match(Set dst (ReverseBytesS src));
7716
7717 ins_cost(INSN_COST);
7718 format %{ "rev16w $dst, $src\n\t"
7719 "sbfmw $dst, $dst, #0, #15" %}
7720
7721 ins_encode %{
7722 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7723 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7724 %}
7725
7726 ins_pipe(ialu_reg);
7727 %}
7728
7729 // ============================================================================
7730 // Zero Count Instructions
7731
7732 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7733 match(Set dst (CountLeadingZerosI src));
7734
7735 ins_cost(INSN_COST);
7736 format %{ "clzw $dst, $src" %}
7737 ins_encode %{
7738 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7739 %}
7740
7741 ins_pipe(ialu_reg);
7742 %}
7743
7744 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7745 match(Set dst (CountLeadingZerosL src));
7746
7747 ins_cost(INSN_COST);
7748 format %{ "clz $dst, $src" %}
7749 ins_encode %{
7750 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7751 %}
7752
7753 ins_pipe(ialu_reg);
7754 %}
7755
7756 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7757 match(Set dst (CountTrailingZerosI src));
7758
7759 ins_cost(INSN_COST * 2);
7760 format %{ "rbitw $dst, $src\n\t"
7761 "clzw $dst, $dst" %}
7762 ins_encode %{
7763 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7764 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7765 %}
7766
7767 ins_pipe(ialu_reg);
7768 %}
7769
7770 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7771 match(Set dst (CountTrailingZerosL src));
7772
7773 ins_cost(INSN_COST * 2);
7774 format %{ "rbit $dst, $src\n\t"
7775 "clz $dst, $dst" %}
7776 ins_encode %{
7777 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7778 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7779 %}
7780
7781 ins_pipe(ialu_reg);
7782 %}
7783
7784 //---------- Population Count Instructions -------------------------------------
7785 //
7786
7787 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7788 match(Set dst (PopCountI src));
7789 effect(TEMP tmp);
7790 ins_cost(INSN_COST * 13);
7791
7792 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7793 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7794 "addv $tmp, $tmp\t# vector (8B)\n\t"
7795 "mov $dst, $tmp\t# vector (1D)" %}
7796 ins_encode %{
7797 __ fmovs($tmp$$FloatRegister, $src$$Register);
7798 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7799 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7800 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7801 %}
7802
7803 ins_pipe(pipe_class_default);
7804 %}
7805
7806 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7807 match(Set dst (PopCountI (LoadI mem)));
7808 effect(TEMP tmp);
7809 ins_cost(INSN_COST * 13);
7810
7811 format %{ "ldrs $tmp, $mem\n\t"
7812 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7813 "addv $tmp, $tmp\t# vector (8B)\n\t"
7814 "mov $dst, $tmp\t# vector (1D)" %}
7815 ins_encode %{
7816 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7817 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7818 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7819 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7820 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7821 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7822 %}
7823
7824 ins_pipe(pipe_class_default);
7825 %}
7826
7827 // Note: Long.bitCount(long) returns an int.
7828 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7829 match(Set dst (PopCountL src));
7830 effect(TEMP tmp);
7831 ins_cost(INSN_COST * 13);
7832
7833 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7834 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7835 "addv $tmp, $tmp\t# vector (8B)\n\t"
7836 "mov $dst, $tmp\t# vector (1D)" %}
7837 ins_encode %{
7838 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7839 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7840 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7841 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7842 %}
7843
7844 ins_pipe(pipe_class_default);
7845 %}
7846
7847 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7848 match(Set dst (PopCountL (LoadL mem)));
7849 effect(TEMP tmp);
7850 ins_cost(INSN_COST * 13);
7851
7852 format %{ "ldrd $tmp, $mem\n\t"
7853 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7854 "addv $tmp, $tmp\t# vector (8B)\n\t"
7855 "mov $dst, $tmp\t# vector (1D)" %}
7856 ins_encode %{
7857 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7858 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7859 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7860 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7861 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7862 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7863 %}
7864
7865 ins_pipe(pipe_class_default);
7866 %}
7867
7868 // ============================================================================
7869 // VerifyVectorAlignment Instruction
7870
7871 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7872 match(Set addr (VerifyVectorAlignment addr mask));
7873 effect(KILL cr);
7874 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7875 ins_encode %{
7876 Label Lskip;
7877 // check if masked bits of addr are zero
7878 __ tst($addr$$Register, $mask$$constant);
7879 __ br(Assembler::EQ, Lskip);
7880 __ stop("verify_vector_alignment found a misaligned vector memory access");
7881 __ bind(Lskip);
7882 %}
7883 ins_pipe(pipe_slow);
7884 %}
7885
7886 // ============================================================================
7887 // MemBar Instruction
7888
7889 instruct load_fence() %{
7890 match(LoadFence);
7891 ins_cost(VOLATILE_REF_COST);
7892
7893 format %{ "load_fence" %}
7894
7895 ins_encode %{
7896 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7897 %}
7898 ins_pipe(pipe_serial);
7899 %}
7900
7901 instruct unnecessary_membar_acquire() %{
7902 predicate(unnecessary_acquire(n));
7903 match(MemBarAcquire);
7904 ins_cost(0);
7905
7906 format %{ "membar_acquire (elided)" %}
7907
7908 ins_encode %{
7909 __ block_comment("membar_acquire (elided)");
7910 %}
7911
7912 ins_pipe(pipe_class_empty);
7913 %}
7914
7915 instruct membar_acquire() %{
7916 match(MemBarAcquire);
7917 ins_cost(VOLATILE_REF_COST);
7918
7919 format %{ "membar_acquire\n\t"
7920 "dmb ishld" %}
7921
7922 ins_encode %{
7923 __ block_comment("membar_acquire");
7924 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7925 %}
7926
7927 ins_pipe(pipe_serial);
7928 %}
7929
7930
7931 instruct membar_acquire_lock() %{
7932 match(MemBarAcquireLock);
7933 ins_cost(VOLATILE_REF_COST);
7934
7935 format %{ "membar_acquire_lock (elided)" %}
7936
7937 ins_encode %{
7938 __ block_comment("membar_acquire_lock (elided)");
7939 %}
7940
7941 ins_pipe(pipe_serial);
7942 %}
7943
7944 instruct store_fence() %{
7945 match(StoreFence);
7946 ins_cost(VOLATILE_REF_COST);
7947
7948 format %{ "store_fence" %}
7949
7950 ins_encode %{
7951 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7952 %}
7953 ins_pipe(pipe_serial);
7954 %}
7955
7956 instruct unnecessary_membar_release() %{
7957 predicate(unnecessary_release(n));
7958 match(MemBarRelease);
7959 ins_cost(0);
7960
7961 format %{ "membar_release (elided)" %}
7962
7963 ins_encode %{
7964 __ block_comment("membar_release (elided)");
7965 %}
7966 ins_pipe(pipe_serial);
7967 %}
7968
7969 instruct membar_release() %{
7970 match(MemBarRelease);
7971 ins_cost(VOLATILE_REF_COST);
7972
7973 format %{ "membar_release\n\t"
7974 "dmb ishst\n\tdmb ishld" %}
7975
7976 ins_encode %{
7977 __ block_comment("membar_release");
7978 // These will be merged if AlwaysMergeDMB is enabled.
7979 __ membar(Assembler::StoreStore);
7980 __ membar(Assembler::LoadStore);
7981 %}
7982 ins_pipe(pipe_serial);
7983 %}
7984
7985 instruct membar_storestore() %{
7986 match(MemBarStoreStore);
7987 match(StoreStoreFence);
7988 ins_cost(VOLATILE_REF_COST);
7989
7990 format %{ "MEMBAR-store-store" %}
7991
7992 ins_encode %{
7993 __ membar(Assembler::StoreStore);
7994 %}
7995 ins_pipe(pipe_serial);
7996 %}
7997
7998 instruct membar_release_lock() %{
7999 match(MemBarReleaseLock);
8000 ins_cost(VOLATILE_REF_COST);
8001
8002 format %{ "membar_release_lock (elided)" %}
8003
8004 ins_encode %{
8005 __ block_comment("membar_release_lock (elided)");
8006 %}
8007
8008 ins_pipe(pipe_serial);
8009 %}
8010
8011 instruct membar_storeload() %{
8012 match(MemBarStoreLoad);
8013 ins_cost(VOLATILE_REF_COST*100);
8014
8015 format %{ "MEMBAR-store-load\n\t"
8016 "dmb ish" %}
8017
8018 ins_encode %{
8019 __ block_comment("membar_storeload");
8020 __ membar(Assembler::StoreLoad);
8021 %}
8022
8023 ins_pipe(pipe_serial);
8024 %}
8025
8026 instruct unnecessary_membar_volatile() %{
8027 predicate(unnecessary_volatile(n));
8028 match(MemBarVolatile);
8029 ins_cost(0);
8030
8031 format %{ "membar_volatile (elided)" %}
8032
8033 ins_encode %{
8034 __ block_comment("membar_volatile (elided)");
8035 %}
8036
8037 ins_pipe(pipe_serial);
8038 %}
8039
8040 instruct membar_volatile() %{
8041 match(MemBarVolatile);
8042 ins_cost(VOLATILE_REF_COST*100);
8043
8044 format %{ "membar_volatile\n\t"
8045 "dmb ish"%}
8046
8047 ins_encode %{
8048 __ block_comment("membar_volatile");
8049 __ membar(Assembler::StoreLoad);
8050 %}
8051
8052 ins_pipe(pipe_serial);
8053 %}
8054
8055 instruct membar_full() %{
8056 match(MemBarFull);
8057 ins_cost(VOLATILE_REF_COST*100);
8058
8059 format %{ "membar_full\n\t"
8060 "dmb ish" %}
8061 ins_encode %{
8062 __ block_comment("membar_full");
8063 __ membar(Assembler::AnyAny);
8064 %}
8065
8066 ins_pipe(pipe_serial);
8067 %}
8068
8069 // ============================================================================
8070 // Cast/Convert Instructions
8071
8072 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8073 match(Set dst (CastX2P src));
8074
8075 ins_cost(INSN_COST);
8076 format %{ "mov $dst, $src\t# long -> ptr" %}
8077
8078 ins_encode %{
8079 if ($dst$$reg != $src$$reg) {
8080 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8081 }
8082 %}
8083
8084 ins_pipe(ialu_reg);
8085 %}
8086
8087 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8088 match(Set dst (CastP2X src));
8089
8090 ins_cost(INSN_COST);
8091 format %{ "mov $dst, $src\t# ptr -> long" %}
8092
8093 ins_encode %{
8094 if ($dst$$reg != $src$$reg) {
8095 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8096 }
8097 %}
8098
8099 ins_pipe(ialu_reg);
8100 %}
8101
8102 // Convert oop into int for vectors alignment masking
8103 instruct convP2I(iRegINoSp dst, iRegP src) %{
8104 match(Set dst (ConvL2I (CastP2X src)));
8105
8106 ins_cost(INSN_COST);
8107 format %{ "movw $dst, $src\t# ptr -> int" %}
8108 ins_encode %{
8109 __ movw($dst$$Register, $src$$Register);
8110 %}
8111
8112 ins_pipe(ialu_reg);
8113 %}
8114
8115 // Convert compressed oop into int for vectors alignment masking
8116 // in case of 32bit oops (heap < 4Gb).
8117 instruct convN2I(iRegINoSp dst, iRegN src)
8118 %{
8119 predicate(CompressedOops::shift() == 0);
8120 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8121
8122 ins_cost(INSN_COST);
8123 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8124 ins_encode %{
8125 __ movw($dst$$Register, $src$$Register);
8126 %}
8127
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131
8132 // Convert oop pointer into compressed form
8133 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8134 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8135 match(Set dst (EncodeP src));
8136 effect(KILL cr);
8137 ins_cost(INSN_COST * 3);
8138 format %{ "encode_heap_oop $dst, $src" %}
8139 ins_encode %{
8140 Register s = $src$$Register;
8141 Register d = $dst$$Register;
8142 __ encode_heap_oop(d, s);
8143 %}
8144 ins_pipe(ialu_reg);
8145 %}
8146
8147 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8148 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8149 match(Set dst (EncodeP src));
8150 ins_cost(INSN_COST * 3);
8151 format %{ "encode_heap_oop_not_null $dst, $src" %}
8152 ins_encode %{
8153 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8154 %}
8155 ins_pipe(ialu_reg);
8156 %}
8157
8158 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8159 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8160 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8161 match(Set dst (DecodeN src));
8162 ins_cost(INSN_COST * 3);
8163 format %{ "decode_heap_oop $dst, $src" %}
8164 ins_encode %{
8165 Register s = $src$$Register;
8166 Register d = $dst$$Register;
8167 __ decode_heap_oop(d, s);
8168 %}
8169 ins_pipe(ialu_reg);
8170 %}
8171
8172 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8173 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8174 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8175 match(Set dst (DecodeN src));
8176 ins_cost(INSN_COST * 3);
8177 format %{ "decode_heap_oop_not_null $dst, $src" %}
8178 ins_encode %{
8179 Register s = $src$$Register;
8180 Register d = $dst$$Register;
8181 __ decode_heap_oop_not_null(d, s);
8182 %}
8183 ins_pipe(ialu_reg);
8184 %}
8185
8186 // n.b. AArch64 implementations of encode_klass_not_null and
8187 // decode_klass_not_null do not modify the flags register so, unlike
8188 // Intel, we don't kill CR as a side effect here
8189
8190 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8191 match(Set dst (EncodePKlass src));
8192
8193 ins_cost(INSN_COST * 3);
8194 format %{ "encode_klass_not_null $dst,$src" %}
8195
8196 ins_encode %{
8197 Register src_reg = as_Register($src$$reg);
8198 Register dst_reg = as_Register($dst$$reg);
8199 __ encode_klass_not_null(dst_reg, src_reg);
8200 %}
8201
8202 ins_pipe(ialu_reg);
8203 %}
8204
8205 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8206 match(Set dst (DecodeNKlass src));
8207
8208 ins_cost(INSN_COST * 3);
8209 format %{ "decode_klass_not_null $dst,$src" %}
8210
8211 ins_encode %{
8212 Register src_reg = as_Register($src$$reg);
8213 Register dst_reg = as_Register($dst$$reg);
8214 if (dst_reg != src_reg) {
8215 __ decode_klass_not_null(dst_reg, src_reg);
8216 } else {
8217 __ decode_klass_not_null(dst_reg);
8218 }
8219 %}
8220
8221 ins_pipe(ialu_reg);
8222 %}
8223
8224 instruct checkCastPP(iRegPNoSp dst)
8225 %{
8226 match(Set dst (CheckCastPP dst));
8227
8228 size(0);
8229 format %{ "# checkcastPP of $dst" %}
8230 ins_encode(/* empty encoding */);
8231 ins_pipe(pipe_class_empty);
8232 %}
8233
8234 instruct castPP(iRegPNoSp dst)
8235 %{
8236 match(Set dst (CastPP dst));
8237
8238 size(0);
8239 format %{ "# castPP of $dst" %}
8240 ins_encode(/* empty encoding */);
8241 ins_pipe(pipe_class_empty);
8242 %}
8243
8244 instruct castII(iRegI dst)
8245 %{
8246 predicate(VerifyConstraintCasts == 0);
8247 match(Set dst (CastII dst));
8248
8249 size(0);
8250 format %{ "# castII of $dst" %}
8251 ins_encode(/* empty encoding */);
8252 ins_cost(0);
8253 ins_pipe(pipe_class_empty);
8254 %}
8255
8256 instruct castII_checked(iRegI dst, rFlagsReg cr)
8257 %{
8258 predicate(VerifyConstraintCasts > 0);
8259 match(Set dst (CastII dst));
8260 effect(KILL cr);
8261
8262 format %{ "# castII_checked of $dst" %}
8263 ins_encode %{
8264 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8265 %}
8266 ins_pipe(pipe_slow);
8267 %}
8268
8269 instruct castLL(iRegL dst)
8270 %{
8271 predicate(VerifyConstraintCasts == 0);
8272 match(Set dst (CastLL dst));
8273
8274 size(0);
8275 format %{ "# castLL of $dst" %}
8276 ins_encode(/* empty encoding */);
8277 ins_cost(0);
8278 ins_pipe(pipe_class_empty);
8279 %}
8280
8281 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8282 %{
8283 predicate(VerifyConstraintCasts > 0);
8284 match(Set dst (CastLL dst));
8285 effect(KILL cr);
8286
8287 format %{ "# castLL_checked of $dst" %}
8288 ins_encode %{
8289 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8290 %}
8291 ins_pipe(pipe_slow);
8292 %}
8293
8294 instruct castHH(vRegF dst)
8295 %{
8296 match(Set dst (CastHH dst));
8297 size(0);
8298 format %{ "# castHH of $dst" %}
8299 ins_encode(/* empty encoding */);
8300 ins_cost(0);
8301 ins_pipe(pipe_class_empty);
8302 %}
8303
8304 instruct castFF(vRegF dst)
8305 %{
8306 match(Set dst (CastFF dst));
8307
8308 size(0);
8309 format %{ "# castFF of $dst" %}
8310 ins_encode(/* empty encoding */);
8311 ins_cost(0);
8312 ins_pipe(pipe_class_empty);
8313 %}
8314
8315 instruct castDD(vRegD dst)
8316 %{
8317 match(Set dst (CastDD dst));
8318
8319 size(0);
8320 format %{ "# castDD of $dst" %}
8321 ins_encode(/* empty encoding */);
8322 ins_cost(0);
8323 ins_pipe(pipe_class_empty);
8324 %}
8325
8326 instruct castVV(vReg dst)
8327 %{
8328 match(Set dst (CastVV dst));
8329
8330 size(0);
8331 format %{ "# castVV of $dst" %}
8332 ins_encode(/* empty encoding */);
8333 ins_cost(0);
8334 ins_pipe(pipe_class_empty);
8335 %}
8336
8337 instruct castVVMask(pRegGov dst)
8338 %{
8339 match(Set dst (CastVV dst));
8340
8341 size(0);
8342 format %{ "# castVV of $dst" %}
8343 ins_encode(/* empty encoding */);
8344 ins_cost(0);
8345 ins_pipe(pipe_class_empty);
8346 %}
8347
8348 // Manifest a CmpU result in an integer register.
8349 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8350 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8351 %{
8352 match(Set dst (CmpU3 src1 src2));
8353 effect(KILL flags);
8354
8355 ins_cost(INSN_COST * 3);
8356 format %{
8357 "cmpw $src1, $src2\n\t"
8358 "csetw $dst, ne\n\t"
8359 "cnegw $dst, lo\t# CmpU3(reg)"
8360 %}
8361 ins_encode %{
8362 __ cmpw($src1$$Register, $src2$$Register);
8363 __ csetw($dst$$Register, Assembler::NE);
8364 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8365 %}
8366
8367 ins_pipe(pipe_class_default);
8368 %}
8369
8370 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8371 %{
8372 match(Set dst (CmpU3 src1 src2));
8373 effect(KILL flags);
8374
8375 ins_cost(INSN_COST * 3);
8376 format %{
8377 "subsw zr, $src1, $src2\n\t"
8378 "csetw $dst, ne\n\t"
8379 "cnegw $dst, lo\t# CmpU3(imm)"
8380 %}
8381 ins_encode %{
8382 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8383 __ csetw($dst$$Register, Assembler::NE);
8384 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8385 %}
8386
8387 ins_pipe(pipe_class_default);
8388 %}
8389
8390 // Manifest a CmpUL result in an integer register.
8391 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8392 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8393 %{
8394 match(Set dst (CmpUL3 src1 src2));
8395 effect(KILL flags);
8396
8397 ins_cost(INSN_COST * 3);
8398 format %{
8399 "cmp $src1, $src2\n\t"
8400 "csetw $dst, ne\n\t"
8401 "cnegw $dst, lo\t# CmpUL3(reg)"
8402 %}
8403 ins_encode %{
8404 __ cmp($src1$$Register, $src2$$Register);
8405 __ csetw($dst$$Register, Assembler::NE);
8406 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8407 %}
8408
8409 ins_pipe(pipe_class_default);
8410 %}
8411
8412 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8413 %{
8414 match(Set dst (CmpUL3 src1 src2));
8415 effect(KILL flags);
8416
8417 ins_cost(INSN_COST * 3);
8418 format %{
8419 "subs zr, $src1, $src2\n\t"
8420 "csetw $dst, ne\n\t"
8421 "cnegw $dst, lo\t# CmpUL3(imm)"
8422 %}
8423 ins_encode %{
8424 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8425 __ csetw($dst$$Register, Assembler::NE);
8426 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8427 %}
8428
8429 ins_pipe(pipe_class_default);
8430 %}
8431
8432 // Manifest a CmpL result in an integer register.
8433 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8434 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8435 %{
8436 match(Set dst (CmpL3 src1 src2));
8437 effect(KILL flags);
8438
8439 ins_cost(INSN_COST * 3);
8440 format %{
8441 "cmp $src1, $src2\n\t"
8442 "csetw $dst, ne\n\t"
8443 "cnegw $dst, lt\t# CmpL3(reg)"
8444 %}
8445 ins_encode %{
8446 __ cmp($src1$$Register, $src2$$Register);
8447 __ csetw($dst$$Register, Assembler::NE);
8448 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8449 %}
8450
8451 ins_pipe(pipe_class_default);
8452 %}
8453
8454 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8455 %{
8456 match(Set dst (CmpL3 src1 src2));
8457 effect(KILL flags);
8458
8459 ins_cost(INSN_COST * 3);
8460 format %{
8461 "subs zr, $src1, $src2\n\t"
8462 "csetw $dst, ne\n\t"
8463 "cnegw $dst, lt\t# CmpL3(imm)"
8464 %}
8465 ins_encode %{
8466 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8467 __ csetw($dst$$Register, Assembler::NE);
8468 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8469 %}
8470
8471 ins_pipe(pipe_class_default);
8472 %}
8473
8474 // ============================================================================
8475 // Conditional Move Instructions
8476
8477 // n.b. we have identical rules for both a signed compare op (cmpOp)
8478 // and an unsigned compare op (cmpOpU). it would be nice if we could
8479 // define an op class which merged both inputs and use it to type the
8480 // argument to a single rule. unfortunatelyt his fails because the
8481 // opclass does not live up to the COND_INTER interface of its
8482 // component operands. When the generic code tries to negate the
8483 // operand it ends up running the generci Machoper::negate method
8484 // which throws a ShouldNotHappen. So, we have to provide two flavours
8485 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8486
8487 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8488 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8489
8490 ins_cost(INSN_COST * 2);
8491 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8492
8493 ins_encode %{
8494 __ cselw(as_Register($dst$$reg),
8495 as_Register($src2$$reg),
8496 as_Register($src1$$reg),
8497 (Assembler::Condition)$cmp$$cmpcode);
8498 %}
8499
8500 ins_pipe(icond_reg_reg);
8501 %}
8502
8503 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8504 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8505
8506 ins_cost(INSN_COST * 2);
8507 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8508
8509 ins_encode %{
8510 __ cselw(as_Register($dst$$reg),
8511 as_Register($src2$$reg),
8512 as_Register($src1$$reg),
8513 (Assembler::Condition)$cmp$$cmpcode);
8514 %}
8515
8516 ins_pipe(icond_reg_reg);
8517 %}
8518
8519 // special cases where one arg is zero
8520
8521 // n.b. this is selected in preference to the rule above because it
8522 // avoids loading constant 0 into a source register
8523
8524 // TODO
8525 // we ought only to be able to cull one of these variants as the ideal
8526 // transforms ought always to order the zero consistently (to left/right?)
8527
8528 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8529 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8530
8531 ins_cost(INSN_COST * 2);
8532 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8533
8534 ins_encode %{
8535 __ cselw(as_Register($dst$$reg),
8536 as_Register($src$$reg),
8537 zr,
8538 (Assembler::Condition)$cmp$$cmpcode);
8539 %}
8540
8541 ins_pipe(icond_reg);
8542 %}
8543
8544 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8545 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8546
8547 ins_cost(INSN_COST * 2);
8548 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8549
8550 ins_encode %{
8551 __ cselw(as_Register($dst$$reg),
8552 as_Register($src$$reg),
8553 zr,
8554 (Assembler::Condition)$cmp$$cmpcode);
8555 %}
8556
8557 ins_pipe(icond_reg);
8558 %}
8559
8560 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8561 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8562
8563 ins_cost(INSN_COST * 2);
8564 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8565
8566 ins_encode %{
8567 __ cselw(as_Register($dst$$reg),
8568 zr,
8569 as_Register($src$$reg),
8570 (Assembler::Condition)$cmp$$cmpcode);
8571 %}
8572
8573 ins_pipe(icond_reg);
8574 %}
8575
8576 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8577 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8578
8579 ins_cost(INSN_COST * 2);
8580 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8581
8582 ins_encode %{
8583 __ cselw(as_Register($dst$$reg),
8584 zr,
8585 as_Register($src$$reg),
8586 (Assembler::Condition)$cmp$$cmpcode);
8587 %}
8588
8589 ins_pipe(icond_reg);
8590 %}
8591
8592 // special case for creating a boolean 0 or 1
8593
8594 // n.b. this is selected in preference to the rule above because it
8595 // avoids loading constants 0 and 1 into a source register
8596
8597 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8598 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8599
8600 ins_cost(INSN_COST * 2);
8601 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8602
8603 ins_encode %{
8604 // equivalently
8605 // cset(as_Register($dst$$reg),
8606 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8607 __ csincw(as_Register($dst$$reg),
8608 zr,
8609 zr,
8610 (Assembler::Condition)$cmp$$cmpcode);
8611 %}
8612
8613 ins_pipe(icond_none);
8614 %}
8615
8616 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8617 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8618
8619 ins_cost(INSN_COST * 2);
8620 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8621
8622 ins_encode %{
8623 // equivalently
8624 // cset(as_Register($dst$$reg),
8625 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8626 __ csincw(as_Register($dst$$reg),
8627 zr,
8628 zr,
8629 (Assembler::Condition)$cmp$$cmpcode);
8630 %}
8631
8632 ins_pipe(icond_none);
8633 %}
8634
8635 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8636 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8637
8638 ins_cost(INSN_COST * 2);
8639 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8640
8641 ins_encode %{
8642 __ csel(as_Register($dst$$reg),
8643 as_Register($src2$$reg),
8644 as_Register($src1$$reg),
8645 (Assembler::Condition)$cmp$$cmpcode);
8646 %}
8647
8648 ins_pipe(icond_reg_reg);
8649 %}
8650
8651 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8652 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8653
8654 ins_cost(INSN_COST * 2);
8655 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8656
8657 ins_encode %{
8658 __ csel(as_Register($dst$$reg),
8659 as_Register($src2$$reg),
8660 as_Register($src1$$reg),
8661 (Assembler::Condition)$cmp$$cmpcode);
8662 %}
8663
8664 ins_pipe(icond_reg_reg);
8665 %}
8666
8667 // special cases where one arg is zero
8668
8669 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8670 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8671
8672 ins_cost(INSN_COST * 2);
8673 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8674
8675 ins_encode %{
8676 __ csel(as_Register($dst$$reg),
8677 zr,
8678 as_Register($src$$reg),
8679 (Assembler::Condition)$cmp$$cmpcode);
8680 %}
8681
8682 ins_pipe(icond_reg);
8683 %}
8684
8685 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8686 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8687
8688 ins_cost(INSN_COST * 2);
8689 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8690
8691 ins_encode %{
8692 __ csel(as_Register($dst$$reg),
8693 zr,
8694 as_Register($src$$reg),
8695 (Assembler::Condition)$cmp$$cmpcode);
8696 %}
8697
8698 ins_pipe(icond_reg);
8699 %}
8700
8701 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8702 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8703
8704 ins_cost(INSN_COST * 2);
8705 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8706
8707 ins_encode %{
8708 __ csel(as_Register($dst$$reg),
8709 as_Register($src$$reg),
8710 zr,
8711 (Assembler::Condition)$cmp$$cmpcode);
8712 %}
8713
8714 ins_pipe(icond_reg);
8715 %}
8716
8717 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8718 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8719
8720 ins_cost(INSN_COST * 2);
8721 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8722
8723 ins_encode %{
8724 __ csel(as_Register($dst$$reg),
8725 as_Register($src$$reg),
8726 zr,
8727 (Assembler::Condition)$cmp$$cmpcode);
8728 %}
8729
8730 ins_pipe(icond_reg);
8731 %}
8732
8733 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8734 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8735
8736 ins_cost(INSN_COST * 2);
8737 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8738
8739 ins_encode %{
8740 __ csel(as_Register($dst$$reg),
8741 as_Register($src2$$reg),
8742 as_Register($src1$$reg),
8743 (Assembler::Condition)$cmp$$cmpcode);
8744 %}
8745
8746 ins_pipe(icond_reg_reg);
8747 %}
8748
8749 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8750 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8751
8752 ins_cost(INSN_COST * 2);
8753 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8754
8755 ins_encode %{
8756 __ csel(as_Register($dst$$reg),
8757 as_Register($src2$$reg),
8758 as_Register($src1$$reg),
8759 (Assembler::Condition)$cmp$$cmpcode);
8760 %}
8761
8762 ins_pipe(icond_reg_reg);
8763 %}
8764
8765 // special cases where one arg is zero
8766
8767 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8768 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8769
8770 ins_cost(INSN_COST * 2);
8771 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8772
8773 ins_encode %{
8774 __ csel(as_Register($dst$$reg),
8775 zr,
8776 as_Register($src$$reg),
8777 (Assembler::Condition)$cmp$$cmpcode);
8778 %}
8779
8780 ins_pipe(icond_reg);
8781 %}
8782
8783 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8784 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8785
8786 ins_cost(INSN_COST * 2);
8787 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8788
8789 ins_encode %{
8790 __ csel(as_Register($dst$$reg),
8791 zr,
8792 as_Register($src$$reg),
8793 (Assembler::Condition)$cmp$$cmpcode);
8794 %}
8795
8796 ins_pipe(icond_reg);
8797 %}
8798
8799 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8800 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8801
8802 ins_cost(INSN_COST * 2);
8803 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8804
8805 ins_encode %{
8806 __ csel(as_Register($dst$$reg),
8807 as_Register($src$$reg),
8808 zr,
8809 (Assembler::Condition)$cmp$$cmpcode);
8810 %}
8811
8812 ins_pipe(icond_reg);
8813 %}
8814
8815 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8816 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8817
8818 ins_cost(INSN_COST * 2);
8819 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8820
8821 ins_encode %{
8822 __ csel(as_Register($dst$$reg),
8823 as_Register($src$$reg),
8824 zr,
8825 (Assembler::Condition)$cmp$$cmpcode);
8826 %}
8827
8828 ins_pipe(icond_reg);
8829 %}
8830
8831 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8832 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8833
8834 ins_cost(INSN_COST * 2);
8835 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8836
8837 ins_encode %{
8838 __ cselw(as_Register($dst$$reg),
8839 as_Register($src2$$reg),
8840 as_Register($src1$$reg),
8841 (Assembler::Condition)$cmp$$cmpcode);
8842 %}
8843
8844 ins_pipe(icond_reg_reg);
8845 %}
8846
8847 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8848 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8849
8850 ins_cost(INSN_COST * 2);
8851 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8852
8853 ins_encode %{
8854 __ cselw(as_Register($dst$$reg),
8855 as_Register($src2$$reg),
8856 as_Register($src1$$reg),
8857 (Assembler::Condition)$cmp$$cmpcode);
8858 %}
8859
8860 ins_pipe(icond_reg_reg);
8861 %}
8862
8863 // special cases where one arg is zero
8864
8865 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8866 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8867
8868 ins_cost(INSN_COST * 2);
8869 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8870
8871 ins_encode %{
8872 __ cselw(as_Register($dst$$reg),
8873 zr,
8874 as_Register($src$$reg),
8875 (Assembler::Condition)$cmp$$cmpcode);
8876 %}
8877
8878 ins_pipe(icond_reg);
8879 %}
8880
8881 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8882 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8883
8884 ins_cost(INSN_COST * 2);
8885 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8886
8887 ins_encode %{
8888 __ cselw(as_Register($dst$$reg),
8889 zr,
8890 as_Register($src$$reg),
8891 (Assembler::Condition)$cmp$$cmpcode);
8892 %}
8893
8894 ins_pipe(icond_reg);
8895 %}
8896
8897 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8898 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8899
8900 ins_cost(INSN_COST * 2);
8901 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8902
8903 ins_encode %{
8904 __ cselw(as_Register($dst$$reg),
8905 as_Register($src$$reg),
8906 zr,
8907 (Assembler::Condition)$cmp$$cmpcode);
8908 %}
8909
8910 ins_pipe(icond_reg);
8911 %}
8912
8913 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8914 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8915
8916 ins_cost(INSN_COST * 2);
8917 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8918
8919 ins_encode %{
8920 __ cselw(as_Register($dst$$reg),
8921 as_Register($src$$reg),
8922 zr,
8923 (Assembler::Condition)$cmp$$cmpcode);
8924 %}
8925
8926 ins_pipe(icond_reg);
8927 %}
8928
8929 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8930 %{
8931 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8932
8933 ins_cost(INSN_COST * 3);
8934
8935 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8936 ins_encode %{
8937 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8938 __ fcsels(as_FloatRegister($dst$$reg),
8939 as_FloatRegister($src2$$reg),
8940 as_FloatRegister($src1$$reg),
8941 cond);
8942 %}
8943
8944 ins_pipe(fp_cond_reg_reg_s);
8945 %}
8946
8947 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8948 %{
8949 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8950
8951 ins_cost(INSN_COST * 3);
8952
8953 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8954 ins_encode %{
8955 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8956 __ fcsels(as_FloatRegister($dst$$reg),
8957 as_FloatRegister($src2$$reg),
8958 as_FloatRegister($src1$$reg),
8959 cond);
8960 %}
8961
8962 ins_pipe(fp_cond_reg_reg_s);
8963 %}
8964
8965 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8966 %{
8967 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8968
8969 ins_cost(INSN_COST * 3);
8970
8971 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8972 ins_encode %{
8973 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8974 __ fcseld(as_FloatRegister($dst$$reg),
8975 as_FloatRegister($src2$$reg),
8976 as_FloatRegister($src1$$reg),
8977 cond);
8978 %}
8979
8980 ins_pipe(fp_cond_reg_reg_d);
8981 %}
8982
8983 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8984 %{
8985 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8986
8987 ins_cost(INSN_COST * 3);
8988
8989 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8990 ins_encode %{
8991 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8992 __ fcseld(as_FloatRegister($dst$$reg),
8993 as_FloatRegister($src2$$reg),
8994 as_FloatRegister($src1$$reg),
8995 cond);
8996 %}
8997
8998 ins_pipe(fp_cond_reg_reg_d);
8999 %}
9000
9001 // ============================================================================
9002 // Arithmetic Instructions
9003 //
9004
9005 // Integer Addition
9006
9007 // TODO
9008 // these currently employ operations which do not set CR and hence are
9009 // not flagged as killing CR but we would like to isolate the cases
9010 // where we want to set flags from those where we don't. need to work
9011 // out how to do that.
9012
9013 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9014 match(Set dst (AddI src1 src2));
9015
9016 ins_cost(INSN_COST);
9017 format %{ "addw $dst, $src1, $src2" %}
9018
9019 ins_encode %{
9020 __ addw(as_Register($dst$$reg),
9021 as_Register($src1$$reg),
9022 as_Register($src2$$reg));
9023 %}
9024
9025 ins_pipe(ialu_reg_reg);
9026 %}
9027
9028 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9029 match(Set dst (AddI src1 src2));
9030
9031 ins_cost(INSN_COST);
9032 format %{ "addw $dst, $src1, $src2" %}
9033
9034 // use opcode to indicate that this is an add not a sub
9035 opcode(0x0);
9036
9037 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9038
9039 ins_pipe(ialu_reg_imm);
9040 %}
9041
9042 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9043 match(Set dst (AddI (ConvL2I src1) src2));
9044
9045 ins_cost(INSN_COST);
9046 format %{ "addw $dst, $src1, $src2" %}
9047
9048 // use opcode to indicate that this is an add not a sub
9049 opcode(0x0);
9050
9051 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9052
9053 ins_pipe(ialu_reg_imm);
9054 %}
9055
9056 // Pointer Addition
9057 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9058 match(Set dst (AddP src1 src2));
9059
9060 ins_cost(INSN_COST);
9061 format %{ "add $dst, $src1, $src2\t# ptr" %}
9062
9063 ins_encode %{
9064 __ add(as_Register($dst$$reg),
9065 as_Register($src1$$reg),
9066 as_Register($src2$$reg));
9067 %}
9068
9069 ins_pipe(ialu_reg_reg);
9070 %}
9071
9072 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9073 match(Set dst (AddP src1 (ConvI2L src2)));
9074
9075 ins_cost(1.9 * INSN_COST);
9076 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9077
9078 ins_encode %{
9079 __ add(as_Register($dst$$reg),
9080 as_Register($src1$$reg),
9081 as_Register($src2$$reg), ext::sxtw);
9082 %}
9083
9084 ins_pipe(ialu_reg_reg);
9085 %}
9086
9087 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9088 match(Set dst (AddP src1 (LShiftL src2 scale)));
9089
9090 ins_cost(1.9 * INSN_COST);
9091 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9092
9093 ins_encode %{
9094 __ lea(as_Register($dst$$reg),
9095 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9096 Address::lsl($scale$$constant)));
9097 %}
9098
9099 ins_pipe(ialu_reg_reg_shift);
9100 %}
9101
9102 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9103 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9104
9105 ins_cost(1.9 * INSN_COST);
9106 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9107
9108 ins_encode %{
9109 __ lea(as_Register($dst$$reg),
9110 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9111 Address::sxtw($scale$$constant)));
9112 %}
9113
9114 ins_pipe(ialu_reg_reg_shift);
9115 %}
9116
9117 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9118 match(Set dst (LShiftL (ConvI2L src) scale));
9119
9120 ins_cost(INSN_COST);
9121 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9122
9123 ins_encode %{
9124 __ sbfiz(as_Register($dst$$reg),
9125 as_Register($src$$reg),
9126 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9127 %}
9128
9129 ins_pipe(ialu_reg_shift);
9130 %}
9131
9132 // Pointer Immediate Addition
9133 // n.b. this needs to be more expensive than using an indirect memory
9134 // operand
9135 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9136 match(Set dst (AddP src1 src2));
9137
9138 ins_cost(INSN_COST);
9139 format %{ "add $dst, $src1, $src2\t# ptr" %}
9140
9141 // use opcode to indicate that this is an add not a sub
9142 opcode(0x0);
9143
9144 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9145
9146 ins_pipe(ialu_reg_imm);
9147 %}
9148
9149 // Long Addition
9150 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9151
9152 match(Set dst (AddL src1 src2));
9153
9154 ins_cost(INSN_COST);
9155 format %{ "add $dst, $src1, $src2" %}
9156
9157 ins_encode %{
9158 __ add(as_Register($dst$$reg),
9159 as_Register($src1$$reg),
9160 as_Register($src2$$reg));
9161 %}
9162
9163 ins_pipe(ialu_reg_reg);
9164 %}
9165
9166 // No constant pool entries requiredLong Immediate Addition.
9167 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9168 match(Set dst (AddL src1 src2));
9169
9170 ins_cost(INSN_COST);
9171 format %{ "add $dst, $src1, $src2" %}
9172
9173 // use opcode to indicate that this is an add not a sub
9174 opcode(0x0);
9175
9176 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9177
9178 ins_pipe(ialu_reg_imm);
9179 %}
9180
9181 // Integer Subtraction
9182 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9183 match(Set dst (SubI src1 src2));
9184
9185 ins_cost(INSN_COST);
9186 format %{ "subw $dst, $src1, $src2" %}
9187
9188 ins_encode %{
9189 __ subw(as_Register($dst$$reg),
9190 as_Register($src1$$reg),
9191 as_Register($src2$$reg));
9192 %}
9193
9194 ins_pipe(ialu_reg_reg);
9195 %}
9196
9197 // Immediate Subtraction
9198 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9199 match(Set dst (SubI src1 src2));
9200
9201 ins_cost(INSN_COST);
9202 format %{ "subw $dst, $src1, $src2" %}
9203
9204 // use opcode to indicate that this is a sub not an add
9205 opcode(0x1);
9206
9207 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9208
9209 ins_pipe(ialu_reg_imm);
9210 %}
9211
9212 // Long Subtraction
9213 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9214
9215 match(Set dst (SubL src1 src2));
9216
9217 ins_cost(INSN_COST);
9218 format %{ "sub $dst, $src1, $src2" %}
9219
9220 ins_encode %{
9221 __ sub(as_Register($dst$$reg),
9222 as_Register($src1$$reg),
9223 as_Register($src2$$reg));
9224 %}
9225
9226 ins_pipe(ialu_reg_reg);
9227 %}
9228
9229 // No constant pool entries requiredLong Immediate Subtraction.
9230 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9231 match(Set dst (SubL src1 src2));
9232
9233 ins_cost(INSN_COST);
9234 format %{ "sub$dst, $src1, $src2" %}
9235
9236 // use opcode to indicate that this is a sub not an add
9237 opcode(0x1);
9238
9239 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9240
9241 ins_pipe(ialu_reg_imm);
9242 %}
9243
9244 // Integer Negation (special case for sub)
9245
9246 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9247 match(Set dst (SubI zero src));
9248
9249 ins_cost(INSN_COST);
9250 format %{ "negw $dst, $src\t# int" %}
9251
9252 ins_encode %{
9253 __ negw(as_Register($dst$$reg),
9254 as_Register($src$$reg));
9255 %}
9256
9257 ins_pipe(ialu_reg);
9258 %}
9259
9260 // Long Negation
9261
9262 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9263 match(Set dst (SubL zero src));
9264
9265 ins_cost(INSN_COST);
9266 format %{ "neg $dst, $src\t# long" %}
9267
9268 ins_encode %{
9269 __ neg(as_Register($dst$$reg),
9270 as_Register($src$$reg));
9271 %}
9272
9273 ins_pipe(ialu_reg);
9274 %}
9275
9276 // Integer Multiply
9277
9278 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9279 match(Set dst (MulI src1 src2));
9280
9281 ins_cost(INSN_COST * 3);
9282 format %{ "mulw $dst, $src1, $src2" %}
9283
9284 ins_encode %{
9285 __ mulw(as_Register($dst$$reg),
9286 as_Register($src1$$reg),
9287 as_Register($src2$$reg));
9288 %}
9289
9290 ins_pipe(imul_reg_reg);
9291 %}
9292
9293 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9294 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9295
9296 ins_cost(INSN_COST * 3);
9297 format %{ "smull $dst, $src1, $src2" %}
9298
9299 ins_encode %{
9300 __ smull(as_Register($dst$$reg),
9301 as_Register($src1$$reg),
9302 as_Register($src2$$reg));
9303 %}
9304
9305 ins_pipe(imul_reg_reg);
9306 %}
9307
9308 // Long Multiply
9309
9310 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9311 match(Set dst (MulL src1 src2));
9312
9313 ins_cost(INSN_COST * 5);
9314 format %{ "mul $dst, $src1, $src2" %}
9315
9316 ins_encode %{
9317 __ mul(as_Register($dst$$reg),
9318 as_Register($src1$$reg),
9319 as_Register($src2$$reg));
9320 %}
9321
9322 ins_pipe(lmul_reg_reg);
9323 %}
9324
9325 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9326 %{
9327 match(Set dst (MulHiL src1 src2));
9328
9329 ins_cost(INSN_COST * 7);
9330 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9331
9332 ins_encode %{
9333 __ smulh(as_Register($dst$$reg),
9334 as_Register($src1$$reg),
9335 as_Register($src2$$reg));
9336 %}
9337
9338 ins_pipe(lmul_reg_reg);
9339 %}
9340
9341 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9342 %{
9343 match(Set dst (UMulHiL src1 src2));
9344
9345 ins_cost(INSN_COST * 7);
9346 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9347
9348 ins_encode %{
9349 __ umulh(as_Register($dst$$reg),
9350 as_Register($src1$$reg),
9351 as_Register($src2$$reg));
9352 %}
9353
9354 ins_pipe(lmul_reg_reg);
9355 %}
9356
9357 // Combined Integer Multiply & Add/Sub
9358
9359 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9360 match(Set dst (AddI src3 (MulI src1 src2)));
9361
9362 ins_cost(INSN_COST * 3);
9363 format %{ "madd $dst, $src1, $src2, $src3" %}
9364
9365 ins_encode %{
9366 __ maddw(as_Register($dst$$reg),
9367 as_Register($src1$$reg),
9368 as_Register($src2$$reg),
9369 as_Register($src3$$reg));
9370 %}
9371
9372 ins_pipe(imac_reg_reg);
9373 %}
9374
9375 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9376 match(Set dst (SubI src3 (MulI src1 src2)));
9377
9378 ins_cost(INSN_COST * 3);
9379 format %{ "msub $dst, $src1, $src2, $src3" %}
9380
9381 ins_encode %{
9382 __ msubw(as_Register($dst$$reg),
9383 as_Register($src1$$reg),
9384 as_Register($src2$$reg),
9385 as_Register($src3$$reg));
9386 %}
9387
9388 ins_pipe(imac_reg_reg);
9389 %}
9390
9391 // Combined Integer Multiply & Neg
9392
9393 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9394 match(Set dst (MulI (SubI zero src1) src2));
9395
9396 ins_cost(INSN_COST * 3);
9397 format %{ "mneg $dst, $src1, $src2" %}
9398
9399 ins_encode %{
9400 __ mnegw(as_Register($dst$$reg),
9401 as_Register($src1$$reg),
9402 as_Register($src2$$reg));
9403 %}
9404
9405 ins_pipe(imac_reg_reg);
9406 %}
9407
9408 // Combined Long Multiply & Add/Sub
9409
9410 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9411 match(Set dst (AddL src3 (MulL src1 src2)));
9412
9413 ins_cost(INSN_COST * 5);
9414 format %{ "madd $dst, $src1, $src2, $src3" %}
9415
9416 ins_encode %{
9417 __ madd(as_Register($dst$$reg),
9418 as_Register($src1$$reg),
9419 as_Register($src2$$reg),
9420 as_Register($src3$$reg));
9421 %}
9422
9423 ins_pipe(lmac_reg_reg);
9424 %}
9425
9426 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9427 match(Set dst (SubL src3 (MulL src1 src2)));
9428
9429 ins_cost(INSN_COST * 5);
9430 format %{ "msub $dst, $src1, $src2, $src3" %}
9431
9432 ins_encode %{
9433 __ msub(as_Register($dst$$reg),
9434 as_Register($src1$$reg),
9435 as_Register($src2$$reg),
9436 as_Register($src3$$reg));
9437 %}
9438
9439 ins_pipe(lmac_reg_reg);
9440 %}
9441
9442 // Combined Long Multiply & Neg
9443
9444 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9445 match(Set dst (MulL (SubL zero src1) src2));
9446
9447 ins_cost(INSN_COST * 5);
9448 format %{ "mneg $dst, $src1, $src2" %}
9449
9450 ins_encode %{
9451 __ mneg(as_Register($dst$$reg),
9452 as_Register($src1$$reg),
9453 as_Register($src2$$reg));
9454 %}
9455
9456 ins_pipe(lmac_reg_reg);
9457 %}
9458
9459 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9460
9461 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9462 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9463
9464 ins_cost(INSN_COST * 3);
9465 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9466
9467 ins_encode %{
9468 __ smaddl(as_Register($dst$$reg),
9469 as_Register($src1$$reg),
9470 as_Register($src2$$reg),
9471 as_Register($src3$$reg));
9472 %}
9473
9474 ins_pipe(imac_reg_reg);
9475 %}
9476
9477 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9478 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9479
9480 ins_cost(INSN_COST * 3);
9481 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9482
9483 ins_encode %{
9484 __ smsubl(as_Register($dst$$reg),
9485 as_Register($src1$$reg),
9486 as_Register($src2$$reg),
9487 as_Register($src3$$reg));
9488 %}
9489
9490 ins_pipe(imac_reg_reg);
9491 %}
9492
9493 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9494 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9495
9496 ins_cost(INSN_COST * 3);
9497 format %{ "smnegl $dst, $src1, $src2" %}
9498
9499 ins_encode %{
9500 __ smnegl(as_Register($dst$$reg),
9501 as_Register($src1$$reg),
9502 as_Register($src2$$reg));
9503 %}
9504
9505 ins_pipe(imac_reg_reg);
9506 %}
9507
9508 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9509
9510 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9511 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9512
9513 ins_cost(INSN_COST * 5);
9514 format %{ "mulw rscratch1, $src1, $src2\n\t"
9515 "maddw $dst, $src3, $src4, rscratch1" %}
9516
9517 ins_encode %{
9518 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9519 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9520
9521 ins_pipe(imac_reg_reg);
9522 %}
9523
9524 // Integer Divide
9525
9526 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9527 match(Set dst (DivI src1 src2));
9528
9529 ins_cost(INSN_COST * 19);
9530 format %{ "sdivw $dst, $src1, $src2" %}
9531
9532 ins_encode(aarch64_enc_divw(dst, src1, src2));
9533 ins_pipe(idiv_reg_reg);
9534 %}
9535
9536 // Long Divide
9537
9538 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9539 match(Set dst (DivL src1 src2));
9540
9541 ins_cost(INSN_COST * 35);
9542 format %{ "sdiv $dst, $src1, $src2" %}
9543
9544 ins_encode(aarch64_enc_div(dst, src1, src2));
9545 ins_pipe(ldiv_reg_reg);
9546 %}
9547
9548 // Integer Remainder
9549
9550 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9551 match(Set dst (ModI src1 src2));
9552
9553 ins_cost(INSN_COST * 22);
9554 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9555 "msubw $dst, rscratch1, $src2, $src1" %}
9556
9557 ins_encode(aarch64_enc_modw(dst, src1, src2));
9558 ins_pipe(idiv_reg_reg);
9559 %}
9560
9561 // Long Remainder
9562
9563 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9564 match(Set dst (ModL src1 src2));
9565
9566 ins_cost(INSN_COST * 38);
9567 format %{ "sdiv rscratch1, $src1, $src2\n"
9568 "msub $dst, rscratch1, $src2, $src1" %}
9569
9570 ins_encode(aarch64_enc_mod(dst, src1, src2));
9571 ins_pipe(ldiv_reg_reg);
9572 %}
9573
9574 // Unsigned Integer Divide
9575
9576 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9577 match(Set dst (UDivI src1 src2));
9578
9579 ins_cost(INSN_COST * 19);
9580 format %{ "udivw $dst, $src1, $src2" %}
9581
9582 ins_encode %{
9583 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9584 %}
9585
9586 ins_pipe(idiv_reg_reg);
9587 %}
9588
9589 // Unsigned Long Divide
9590
9591 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9592 match(Set dst (UDivL src1 src2));
9593
9594 ins_cost(INSN_COST * 35);
9595 format %{ "udiv $dst, $src1, $src2" %}
9596
9597 ins_encode %{
9598 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9599 %}
9600
9601 ins_pipe(ldiv_reg_reg);
9602 %}
9603
9604 // Unsigned Integer Remainder
9605
9606 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9607 match(Set dst (UModI src1 src2));
9608
9609 ins_cost(INSN_COST * 22);
9610 format %{ "udivw rscratch1, $src1, $src2\n\t"
9611 "msubw $dst, rscratch1, $src2, $src1" %}
9612
9613 ins_encode %{
9614 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9615 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9616 %}
9617
9618 ins_pipe(idiv_reg_reg);
9619 %}
9620
9621 // Unsigned Long Remainder
9622
9623 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9624 match(Set dst (UModL src1 src2));
9625
9626 ins_cost(INSN_COST * 38);
9627 format %{ "udiv rscratch1, $src1, $src2\n"
9628 "msub $dst, rscratch1, $src2, $src1" %}
9629
9630 ins_encode %{
9631 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9632 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9633 %}
9634
9635 ins_pipe(ldiv_reg_reg);
9636 %}
9637
9638 // Integer Shifts
9639
9640 // Shift Left Register
9641 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9642 match(Set dst (LShiftI src1 src2));
9643
9644 ins_cost(INSN_COST * 2);
9645 format %{ "lslvw $dst, $src1, $src2" %}
9646
9647 ins_encode %{
9648 __ lslvw(as_Register($dst$$reg),
9649 as_Register($src1$$reg),
9650 as_Register($src2$$reg));
9651 %}
9652
9653 ins_pipe(ialu_reg_reg_vshift);
9654 %}
9655
9656 // Shift Left Immediate
9657 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9658 match(Set dst (LShiftI src1 src2));
9659
9660 ins_cost(INSN_COST);
9661 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9662
9663 ins_encode %{
9664 __ lslw(as_Register($dst$$reg),
9665 as_Register($src1$$reg),
9666 $src2$$constant & 0x1f);
9667 %}
9668
9669 ins_pipe(ialu_reg_shift);
9670 %}
9671
9672 // Shift Right Logical Register
9673 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9674 match(Set dst (URShiftI src1 src2));
9675
9676 ins_cost(INSN_COST * 2);
9677 format %{ "lsrvw $dst, $src1, $src2" %}
9678
9679 ins_encode %{
9680 __ lsrvw(as_Register($dst$$reg),
9681 as_Register($src1$$reg),
9682 as_Register($src2$$reg));
9683 %}
9684
9685 ins_pipe(ialu_reg_reg_vshift);
9686 %}
9687
9688 // Shift Right Logical Immediate
9689 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9690 match(Set dst (URShiftI src1 src2));
9691
9692 ins_cost(INSN_COST);
9693 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9694
9695 ins_encode %{
9696 __ lsrw(as_Register($dst$$reg),
9697 as_Register($src1$$reg),
9698 $src2$$constant & 0x1f);
9699 %}
9700
9701 ins_pipe(ialu_reg_shift);
9702 %}
9703
9704 // Shift Right Arithmetic Register
9705 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9706 match(Set dst (RShiftI src1 src2));
9707
9708 ins_cost(INSN_COST * 2);
9709 format %{ "asrvw $dst, $src1, $src2" %}
9710
9711 ins_encode %{
9712 __ asrvw(as_Register($dst$$reg),
9713 as_Register($src1$$reg),
9714 as_Register($src2$$reg));
9715 %}
9716
9717 ins_pipe(ialu_reg_reg_vshift);
9718 %}
9719
9720 // Shift Right Arithmetic Immediate
9721 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9722 match(Set dst (RShiftI src1 src2));
9723
9724 ins_cost(INSN_COST);
9725 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9726
9727 ins_encode %{
9728 __ asrw(as_Register($dst$$reg),
9729 as_Register($src1$$reg),
9730 $src2$$constant & 0x1f);
9731 %}
9732
9733 ins_pipe(ialu_reg_shift);
9734 %}
9735
9736 // Combined Int Mask and Right Shift (using UBFM)
9737 // TODO
9738
9739 // Long Shifts
9740
9741 // Shift Left Register
9742 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9743 match(Set dst (LShiftL src1 src2));
9744
9745 ins_cost(INSN_COST * 2);
9746 format %{ "lslv $dst, $src1, $src2" %}
9747
9748 ins_encode %{
9749 __ lslv(as_Register($dst$$reg),
9750 as_Register($src1$$reg),
9751 as_Register($src2$$reg));
9752 %}
9753
9754 ins_pipe(ialu_reg_reg_vshift);
9755 %}
9756
9757 // Shift Left Immediate
9758 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9759 match(Set dst (LShiftL src1 src2));
9760
9761 ins_cost(INSN_COST);
9762 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9763
9764 ins_encode %{
9765 __ lsl(as_Register($dst$$reg),
9766 as_Register($src1$$reg),
9767 $src2$$constant & 0x3f);
9768 %}
9769
9770 ins_pipe(ialu_reg_shift);
9771 %}
9772
9773 // Shift Right Logical Register
9774 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9775 match(Set dst (URShiftL src1 src2));
9776
9777 ins_cost(INSN_COST * 2);
9778 format %{ "lsrv $dst, $src1, $src2" %}
9779
9780 ins_encode %{
9781 __ lsrv(as_Register($dst$$reg),
9782 as_Register($src1$$reg),
9783 as_Register($src2$$reg));
9784 %}
9785
9786 ins_pipe(ialu_reg_reg_vshift);
9787 %}
9788
9789 // Shift Right Logical Immediate
9790 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9791 match(Set dst (URShiftL src1 src2));
9792
9793 ins_cost(INSN_COST);
9794 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9795
9796 ins_encode %{
9797 __ lsr(as_Register($dst$$reg),
9798 as_Register($src1$$reg),
9799 $src2$$constant & 0x3f);
9800 %}
9801
9802 ins_pipe(ialu_reg_shift);
9803 %}
9804
9805 // A special-case pattern for card table stores.
9806 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9807 match(Set dst (URShiftL (CastP2X src1) src2));
9808
9809 ins_cost(INSN_COST);
9810 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9811
9812 ins_encode %{
9813 __ lsr(as_Register($dst$$reg),
9814 as_Register($src1$$reg),
9815 $src2$$constant & 0x3f);
9816 %}
9817
9818 ins_pipe(ialu_reg_shift);
9819 %}
9820
9821 // Shift Right Arithmetic Register
9822 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9823 match(Set dst (RShiftL src1 src2));
9824
9825 ins_cost(INSN_COST * 2);
9826 format %{ "asrv $dst, $src1, $src2" %}
9827
9828 ins_encode %{
9829 __ asrv(as_Register($dst$$reg),
9830 as_Register($src1$$reg),
9831 as_Register($src2$$reg));
9832 %}
9833
9834 ins_pipe(ialu_reg_reg_vshift);
9835 %}
9836
9837 // Shift Right Arithmetic Immediate
9838 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9839 match(Set dst (RShiftL src1 src2));
9840
9841 ins_cost(INSN_COST);
9842 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9843
9844 ins_encode %{
9845 __ asr(as_Register($dst$$reg),
9846 as_Register($src1$$reg),
9847 $src2$$constant & 0x3f);
9848 %}
9849
9850 ins_pipe(ialu_reg_shift);
9851 %}
9852
9853 // BEGIN This section of the file is automatically generated. Do not edit --------------
9854 // This section is generated from aarch64_ad.m4
9855
9856 // This pattern is automatically generated from aarch64_ad.m4
9857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9858 instruct regL_not_reg(iRegLNoSp dst,
9859 iRegL src1, immL_M1 m1,
9860 rFlagsReg cr) %{
9861 match(Set dst (XorL src1 m1));
9862 ins_cost(INSN_COST);
9863 format %{ "eon $dst, $src1, zr" %}
9864
9865 ins_encode %{
9866 __ eon(as_Register($dst$$reg),
9867 as_Register($src1$$reg),
9868 zr,
9869 Assembler::LSL, 0);
9870 %}
9871
9872 ins_pipe(ialu_reg);
9873 %}
9874
9875 // This pattern is automatically generated from aarch64_ad.m4
9876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9877 instruct regI_not_reg(iRegINoSp dst,
9878 iRegIorL2I src1, immI_M1 m1,
9879 rFlagsReg cr) %{
9880 match(Set dst (XorI src1 m1));
9881 ins_cost(INSN_COST);
9882 format %{ "eonw $dst, $src1, zr" %}
9883
9884 ins_encode %{
9885 __ eonw(as_Register($dst$$reg),
9886 as_Register($src1$$reg),
9887 zr,
9888 Assembler::LSL, 0);
9889 %}
9890
9891 ins_pipe(ialu_reg);
9892 %}
9893
9894 // This pattern is automatically generated from aarch64_ad.m4
9895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9896 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9897 immI0 zero, iRegIorL2I src1, immI src2) %{
9898 match(Set dst (SubI zero (URShiftI src1 src2)));
9899
9900 ins_cost(1.9 * INSN_COST);
9901 format %{ "negw $dst, $src1, LSR $src2" %}
9902
9903 ins_encode %{
9904 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9905 Assembler::LSR, $src2$$constant & 0x1f);
9906 %}
9907
9908 ins_pipe(ialu_reg_shift);
9909 %}
9910
9911 // This pattern is automatically generated from aarch64_ad.m4
9912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9913 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9914 immI0 zero, iRegIorL2I src1, immI src2) %{
9915 match(Set dst (SubI zero (RShiftI src1 src2)));
9916
9917 ins_cost(1.9 * INSN_COST);
9918 format %{ "negw $dst, $src1, ASR $src2" %}
9919
9920 ins_encode %{
9921 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9922 Assembler::ASR, $src2$$constant & 0x1f);
9923 %}
9924
9925 ins_pipe(ialu_reg_shift);
9926 %}
9927
9928 // This pattern is automatically generated from aarch64_ad.m4
9929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9930 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9931 immI0 zero, iRegIorL2I src1, immI src2) %{
9932 match(Set dst (SubI zero (LShiftI src1 src2)));
9933
9934 ins_cost(1.9 * INSN_COST);
9935 format %{ "negw $dst, $src1, LSL $src2" %}
9936
9937 ins_encode %{
9938 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9939 Assembler::LSL, $src2$$constant & 0x1f);
9940 %}
9941
9942 ins_pipe(ialu_reg_shift);
9943 %}
9944
9945 // This pattern is automatically generated from aarch64_ad.m4
9946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9947 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9948 immL0 zero, iRegL src1, immI src2) %{
9949 match(Set dst (SubL zero (URShiftL src1 src2)));
9950
9951 ins_cost(1.9 * INSN_COST);
9952 format %{ "neg $dst, $src1, LSR $src2" %}
9953
9954 ins_encode %{
9955 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9956 Assembler::LSR, $src2$$constant & 0x3f);
9957 %}
9958
9959 ins_pipe(ialu_reg_shift);
9960 %}
9961
9962 // This pattern is automatically generated from aarch64_ad.m4
9963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9964 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9965 immL0 zero, iRegL src1, immI src2) %{
9966 match(Set dst (SubL zero (RShiftL src1 src2)));
9967
9968 ins_cost(1.9 * INSN_COST);
9969 format %{ "neg $dst, $src1, ASR $src2" %}
9970
9971 ins_encode %{
9972 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9973 Assembler::ASR, $src2$$constant & 0x3f);
9974 %}
9975
9976 ins_pipe(ialu_reg_shift);
9977 %}
9978
9979 // This pattern is automatically generated from aarch64_ad.m4
9980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9981 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9982 immL0 zero, iRegL src1, immI src2) %{
9983 match(Set dst (SubL zero (LShiftL src1 src2)));
9984
9985 ins_cost(1.9 * INSN_COST);
9986 format %{ "neg $dst, $src1, LSL $src2" %}
9987
9988 ins_encode %{
9989 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9990 Assembler::LSL, $src2$$constant & 0x3f);
9991 %}
9992
9993 ins_pipe(ialu_reg_shift);
9994 %}
9995
9996 // This pattern is automatically generated from aarch64_ad.m4
9997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9998 instruct AndI_reg_not_reg(iRegINoSp dst,
9999 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10000 match(Set dst (AndI src1 (XorI src2 m1)));
10001 ins_cost(INSN_COST);
10002 format %{ "bicw $dst, $src1, $src2" %}
10003
10004 ins_encode %{
10005 __ bicw(as_Register($dst$$reg),
10006 as_Register($src1$$reg),
10007 as_Register($src2$$reg),
10008 Assembler::LSL, 0);
10009 %}
10010
10011 ins_pipe(ialu_reg_reg);
10012 %}
10013
10014 // This pattern is automatically generated from aarch64_ad.m4
10015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10016 instruct AndL_reg_not_reg(iRegLNoSp dst,
10017 iRegL src1, iRegL src2, immL_M1 m1) %{
10018 match(Set dst (AndL src1 (XorL src2 m1)));
10019 ins_cost(INSN_COST);
10020 format %{ "bic $dst, $src1, $src2" %}
10021
10022 ins_encode %{
10023 __ bic(as_Register($dst$$reg),
10024 as_Register($src1$$reg),
10025 as_Register($src2$$reg),
10026 Assembler::LSL, 0);
10027 %}
10028
10029 ins_pipe(ialu_reg_reg);
10030 %}
10031
10032 // This pattern is automatically generated from aarch64_ad.m4
10033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10034 instruct OrI_reg_not_reg(iRegINoSp dst,
10035 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10036 match(Set dst (OrI src1 (XorI src2 m1)));
10037 ins_cost(INSN_COST);
10038 format %{ "ornw $dst, $src1, $src2" %}
10039
10040 ins_encode %{
10041 __ ornw(as_Register($dst$$reg),
10042 as_Register($src1$$reg),
10043 as_Register($src2$$reg),
10044 Assembler::LSL, 0);
10045 %}
10046
10047 ins_pipe(ialu_reg_reg);
10048 %}
10049
10050 // This pattern is automatically generated from aarch64_ad.m4
10051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10052 instruct OrL_reg_not_reg(iRegLNoSp dst,
10053 iRegL src1, iRegL src2, immL_M1 m1) %{
10054 match(Set dst (OrL src1 (XorL src2 m1)));
10055 ins_cost(INSN_COST);
10056 format %{ "orn $dst, $src1, $src2" %}
10057
10058 ins_encode %{
10059 __ orn(as_Register($dst$$reg),
10060 as_Register($src1$$reg),
10061 as_Register($src2$$reg),
10062 Assembler::LSL, 0);
10063 %}
10064
10065 ins_pipe(ialu_reg_reg);
10066 %}
10067
10068 // This pattern is automatically generated from aarch64_ad.m4
10069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10070 instruct XorI_reg_not_reg(iRegINoSp dst,
10071 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10072 match(Set dst (XorI m1 (XorI src2 src1)));
10073 ins_cost(INSN_COST);
10074 format %{ "eonw $dst, $src1, $src2" %}
10075
10076 ins_encode %{
10077 __ eonw(as_Register($dst$$reg),
10078 as_Register($src1$$reg),
10079 as_Register($src2$$reg),
10080 Assembler::LSL, 0);
10081 %}
10082
10083 ins_pipe(ialu_reg_reg);
10084 %}
10085
10086 // This pattern is automatically generated from aarch64_ad.m4
10087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10088 instruct XorL_reg_not_reg(iRegLNoSp dst,
10089 iRegL src1, iRegL src2, immL_M1 m1) %{
10090 match(Set dst (XorL m1 (XorL src2 src1)));
10091 ins_cost(INSN_COST);
10092 format %{ "eon $dst, $src1, $src2" %}
10093
10094 ins_encode %{
10095 __ eon(as_Register($dst$$reg),
10096 as_Register($src1$$reg),
10097 as_Register($src2$$reg),
10098 Assembler::LSL, 0);
10099 %}
10100
10101 ins_pipe(ialu_reg_reg);
10102 %}
10103
10104 // This pattern is automatically generated from aarch64_ad.m4
10105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10106 // val & (-1 ^ (val >>> shift)) ==> bicw
10107 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10108 iRegIorL2I src1, iRegIorL2I src2,
10109 immI src3, immI_M1 src4) %{
10110 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10111 ins_cost(1.9 * INSN_COST);
10112 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10113
10114 ins_encode %{
10115 __ bicw(as_Register($dst$$reg),
10116 as_Register($src1$$reg),
10117 as_Register($src2$$reg),
10118 Assembler::LSR,
10119 $src3$$constant & 0x1f);
10120 %}
10121
10122 ins_pipe(ialu_reg_reg_shift);
10123 %}
10124
10125 // This pattern is automatically generated from aarch64_ad.m4
10126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10127 // val & (-1 ^ (val >>> shift)) ==> bic
10128 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10129 iRegL src1, iRegL src2,
10130 immI src3, immL_M1 src4) %{
10131 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10132 ins_cost(1.9 * INSN_COST);
10133 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10134
10135 ins_encode %{
10136 __ bic(as_Register($dst$$reg),
10137 as_Register($src1$$reg),
10138 as_Register($src2$$reg),
10139 Assembler::LSR,
10140 $src3$$constant & 0x3f);
10141 %}
10142
10143 ins_pipe(ialu_reg_reg_shift);
10144 %}
10145
10146 // This pattern is automatically generated from aarch64_ad.m4
10147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10148 // val & (-1 ^ (val >> shift)) ==> bicw
10149 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10150 iRegIorL2I src1, iRegIorL2I src2,
10151 immI src3, immI_M1 src4) %{
10152 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10153 ins_cost(1.9 * INSN_COST);
10154 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10155
10156 ins_encode %{
10157 __ bicw(as_Register($dst$$reg),
10158 as_Register($src1$$reg),
10159 as_Register($src2$$reg),
10160 Assembler::ASR,
10161 $src3$$constant & 0x1f);
10162 %}
10163
10164 ins_pipe(ialu_reg_reg_shift);
10165 %}
10166
10167 // This pattern is automatically generated from aarch64_ad.m4
10168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10169 // val & (-1 ^ (val >> shift)) ==> bic
10170 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10171 iRegL src1, iRegL src2,
10172 immI src3, immL_M1 src4) %{
10173 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10174 ins_cost(1.9 * INSN_COST);
10175 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10176
10177 ins_encode %{
10178 __ bic(as_Register($dst$$reg),
10179 as_Register($src1$$reg),
10180 as_Register($src2$$reg),
10181 Assembler::ASR,
10182 $src3$$constant & 0x3f);
10183 %}
10184
10185 ins_pipe(ialu_reg_reg_shift);
10186 %}
10187
10188 // This pattern is automatically generated from aarch64_ad.m4
10189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10190 // val & (-1 ^ (val ror shift)) ==> bicw
10191 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10192 iRegIorL2I src1, iRegIorL2I src2,
10193 immI src3, immI_M1 src4) %{
10194 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10195 ins_cost(1.9 * INSN_COST);
10196 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10197
10198 ins_encode %{
10199 __ bicw(as_Register($dst$$reg),
10200 as_Register($src1$$reg),
10201 as_Register($src2$$reg),
10202 Assembler::ROR,
10203 $src3$$constant & 0x1f);
10204 %}
10205
10206 ins_pipe(ialu_reg_reg_shift);
10207 %}
10208
10209 // This pattern is automatically generated from aarch64_ad.m4
10210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10211 // val & (-1 ^ (val ror shift)) ==> bic
10212 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10213 iRegL src1, iRegL src2,
10214 immI src3, immL_M1 src4) %{
10215 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10216 ins_cost(1.9 * INSN_COST);
10217 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10218
10219 ins_encode %{
10220 __ bic(as_Register($dst$$reg),
10221 as_Register($src1$$reg),
10222 as_Register($src2$$reg),
10223 Assembler::ROR,
10224 $src3$$constant & 0x3f);
10225 %}
10226
10227 ins_pipe(ialu_reg_reg_shift);
10228 %}
10229
10230 // This pattern is automatically generated from aarch64_ad.m4
10231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10232 // val & (-1 ^ (val << shift)) ==> bicw
10233 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10234 iRegIorL2I src1, iRegIorL2I src2,
10235 immI src3, immI_M1 src4) %{
10236 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10237 ins_cost(1.9 * INSN_COST);
10238 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10239
10240 ins_encode %{
10241 __ bicw(as_Register($dst$$reg),
10242 as_Register($src1$$reg),
10243 as_Register($src2$$reg),
10244 Assembler::LSL,
10245 $src3$$constant & 0x1f);
10246 %}
10247
10248 ins_pipe(ialu_reg_reg_shift);
10249 %}
10250
10251 // This pattern is automatically generated from aarch64_ad.m4
10252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10253 // val & (-1 ^ (val << shift)) ==> bic
10254 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10255 iRegL src1, iRegL src2,
10256 immI src3, immL_M1 src4) %{
10257 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10258 ins_cost(1.9 * INSN_COST);
10259 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10260
10261 ins_encode %{
10262 __ bic(as_Register($dst$$reg),
10263 as_Register($src1$$reg),
10264 as_Register($src2$$reg),
10265 Assembler::LSL,
10266 $src3$$constant & 0x3f);
10267 %}
10268
10269 ins_pipe(ialu_reg_reg_shift);
10270 %}
10271
10272 // This pattern is automatically generated from aarch64_ad.m4
10273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10274 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10275 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10276 iRegIorL2I src1, iRegIorL2I src2,
10277 immI src3, immI_M1 src4) %{
10278 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10279 ins_cost(1.9 * INSN_COST);
10280 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10281
10282 ins_encode %{
10283 __ eonw(as_Register($dst$$reg),
10284 as_Register($src1$$reg),
10285 as_Register($src2$$reg),
10286 Assembler::LSR,
10287 $src3$$constant & 0x1f);
10288 %}
10289
10290 ins_pipe(ialu_reg_reg_shift);
10291 %}
10292
10293 // This pattern is automatically generated from aarch64_ad.m4
10294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10295 // val ^ (-1 ^ (val >>> shift)) ==> eon
10296 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10297 iRegL src1, iRegL src2,
10298 immI src3, immL_M1 src4) %{
10299 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10300 ins_cost(1.9 * INSN_COST);
10301 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10302
10303 ins_encode %{
10304 __ eon(as_Register($dst$$reg),
10305 as_Register($src1$$reg),
10306 as_Register($src2$$reg),
10307 Assembler::LSR,
10308 $src3$$constant & 0x3f);
10309 %}
10310
10311 ins_pipe(ialu_reg_reg_shift);
10312 %}
10313
10314 // This pattern is automatically generated from aarch64_ad.m4
10315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10316 // val ^ (-1 ^ (val >> shift)) ==> eonw
10317 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10318 iRegIorL2I src1, iRegIorL2I src2,
10319 immI src3, immI_M1 src4) %{
10320 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10321 ins_cost(1.9 * INSN_COST);
10322 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10323
10324 ins_encode %{
10325 __ eonw(as_Register($dst$$reg),
10326 as_Register($src1$$reg),
10327 as_Register($src2$$reg),
10328 Assembler::ASR,
10329 $src3$$constant & 0x1f);
10330 %}
10331
10332 ins_pipe(ialu_reg_reg_shift);
10333 %}
10334
10335 // This pattern is automatically generated from aarch64_ad.m4
10336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10337 // val ^ (-1 ^ (val >> shift)) ==> eon
10338 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10339 iRegL src1, iRegL src2,
10340 immI src3, immL_M1 src4) %{
10341 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10342 ins_cost(1.9 * INSN_COST);
10343 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10344
10345 ins_encode %{
10346 __ eon(as_Register($dst$$reg),
10347 as_Register($src1$$reg),
10348 as_Register($src2$$reg),
10349 Assembler::ASR,
10350 $src3$$constant & 0x3f);
10351 %}
10352
10353 ins_pipe(ialu_reg_reg_shift);
10354 %}
10355
10356 // This pattern is automatically generated from aarch64_ad.m4
10357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10358 // val ^ (-1 ^ (val ror shift)) ==> eonw
10359 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10360 iRegIorL2I src1, iRegIorL2I src2,
10361 immI src3, immI_M1 src4) %{
10362 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10363 ins_cost(1.9 * INSN_COST);
10364 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10365
10366 ins_encode %{
10367 __ eonw(as_Register($dst$$reg),
10368 as_Register($src1$$reg),
10369 as_Register($src2$$reg),
10370 Assembler::ROR,
10371 $src3$$constant & 0x1f);
10372 %}
10373
10374 ins_pipe(ialu_reg_reg_shift);
10375 %}
10376
10377 // This pattern is automatically generated from aarch64_ad.m4
10378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10379 // val ^ (-1 ^ (val ror shift)) ==> eon
10380 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10381 iRegL src1, iRegL src2,
10382 immI src3, immL_M1 src4) %{
10383 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10384 ins_cost(1.9 * INSN_COST);
10385 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10386
10387 ins_encode %{
10388 __ eon(as_Register($dst$$reg),
10389 as_Register($src1$$reg),
10390 as_Register($src2$$reg),
10391 Assembler::ROR,
10392 $src3$$constant & 0x3f);
10393 %}
10394
10395 ins_pipe(ialu_reg_reg_shift);
10396 %}
10397
10398 // This pattern is automatically generated from aarch64_ad.m4
10399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10400 // val ^ (-1 ^ (val << shift)) ==> eonw
10401 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10402 iRegIorL2I src1, iRegIorL2I src2,
10403 immI src3, immI_M1 src4) %{
10404 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10405 ins_cost(1.9 * INSN_COST);
10406 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10407
10408 ins_encode %{
10409 __ eonw(as_Register($dst$$reg),
10410 as_Register($src1$$reg),
10411 as_Register($src2$$reg),
10412 Assembler::LSL,
10413 $src3$$constant & 0x1f);
10414 %}
10415
10416 ins_pipe(ialu_reg_reg_shift);
10417 %}
10418
10419 // This pattern is automatically generated from aarch64_ad.m4
10420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10421 // val ^ (-1 ^ (val << shift)) ==> eon
10422 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10423 iRegL src1, iRegL src2,
10424 immI src3, immL_M1 src4) %{
10425 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10426 ins_cost(1.9 * INSN_COST);
10427 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10428
10429 ins_encode %{
10430 __ eon(as_Register($dst$$reg),
10431 as_Register($src1$$reg),
10432 as_Register($src2$$reg),
10433 Assembler::LSL,
10434 $src3$$constant & 0x3f);
10435 %}
10436
10437 ins_pipe(ialu_reg_reg_shift);
10438 %}
10439
10440 // This pattern is automatically generated from aarch64_ad.m4
10441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10442 // val | (-1 ^ (val >>> shift)) ==> ornw
10443 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10444 iRegIorL2I src1, iRegIorL2I src2,
10445 immI src3, immI_M1 src4) %{
10446 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10447 ins_cost(1.9 * INSN_COST);
10448 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10449
10450 ins_encode %{
10451 __ ornw(as_Register($dst$$reg),
10452 as_Register($src1$$reg),
10453 as_Register($src2$$reg),
10454 Assembler::LSR,
10455 $src3$$constant & 0x1f);
10456 %}
10457
10458 ins_pipe(ialu_reg_reg_shift);
10459 %}
10460
10461 // This pattern is automatically generated from aarch64_ad.m4
10462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10463 // val | (-1 ^ (val >>> shift)) ==> orn
10464 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10465 iRegL src1, iRegL src2,
10466 immI src3, immL_M1 src4) %{
10467 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10468 ins_cost(1.9 * INSN_COST);
10469 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10470
10471 ins_encode %{
10472 __ orn(as_Register($dst$$reg),
10473 as_Register($src1$$reg),
10474 as_Register($src2$$reg),
10475 Assembler::LSR,
10476 $src3$$constant & 0x3f);
10477 %}
10478
10479 ins_pipe(ialu_reg_reg_shift);
10480 %}
10481
10482 // This pattern is automatically generated from aarch64_ad.m4
10483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10484 // val | (-1 ^ (val >> shift)) ==> ornw
10485 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10486 iRegIorL2I src1, iRegIorL2I src2,
10487 immI src3, immI_M1 src4) %{
10488 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10489 ins_cost(1.9 * INSN_COST);
10490 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10491
10492 ins_encode %{
10493 __ ornw(as_Register($dst$$reg),
10494 as_Register($src1$$reg),
10495 as_Register($src2$$reg),
10496 Assembler::ASR,
10497 $src3$$constant & 0x1f);
10498 %}
10499
10500 ins_pipe(ialu_reg_reg_shift);
10501 %}
10502
10503 // This pattern is automatically generated from aarch64_ad.m4
10504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10505 // val | (-1 ^ (val >> shift)) ==> orn
10506 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10507 iRegL src1, iRegL src2,
10508 immI src3, immL_M1 src4) %{
10509 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10510 ins_cost(1.9 * INSN_COST);
10511 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10512
10513 ins_encode %{
10514 __ orn(as_Register($dst$$reg),
10515 as_Register($src1$$reg),
10516 as_Register($src2$$reg),
10517 Assembler::ASR,
10518 $src3$$constant & 0x3f);
10519 %}
10520
10521 ins_pipe(ialu_reg_reg_shift);
10522 %}
10523
10524 // This pattern is automatically generated from aarch64_ad.m4
10525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10526 // val | (-1 ^ (val ror shift)) ==> ornw
10527 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10528 iRegIorL2I src1, iRegIorL2I src2,
10529 immI src3, immI_M1 src4) %{
10530 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10531 ins_cost(1.9 * INSN_COST);
10532 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10533
10534 ins_encode %{
10535 __ ornw(as_Register($dst$$reg),
10536 as_Register($src1$$reg),
10537 as_Register($src2$$reg),
10538 Assembler::ROR,
10539 $src3$$constant & 0x1f);
10540 %}
10541
10542 ins_pipe(ialu_reg_reg_shift);
10543 %}
10544
10545 // This pattern is automatically generated from aarch64_ad.m4
10546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10547 // val | (-1 ^ (val ror shift)) ==> orn
10548 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10549 iRegL src1, iRegL src2,
10550 immI src3, immL_M1 src4) %{
10551 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10552 ins_cost(1.9 * INSN_COST);
10553 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10554
10555 ins_encode %{
10556 __ orn(as_Register($dst$$reg),
10557 as_Register($src1$$reg),
10558 as_Register($src2$$reg),
10559 Assembler::ROR,
10560 $src3$$constant & 0x3f);
10561 %}
10562
10563 ins_pipe(ialu_reg_reg_shift);
10564 %}
10565
10566 // This pattern is automatically generated from aarch64_ad.m4
10567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10568 // val | (-1 ^ (val << shift)) ==> ornw
10569 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10570 iRegIorL2I src1, iRegIorL2I src2,
10571 immI src3, immI_M1 src4) %{
10572 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10573 ins_cost(1.9 * INSN_COST);
10574 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10575
10576 ins_encode %{
10577 __ ornw(as_Register($dst$$reg),
10578 as_Register($src1$$reg),
10579 as_Register($src2$$reg),
10580 Assembler::LSL,
10581 $src3$$constant & 0x1f);
10582 %}
10583
10584 ins_pipe(ialu_reg_reg_shift);
10585 %}
10586
10587 // This pattern is automatically generated from aarch64_ad.m4
10588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10589 // val | (-1 ^ (val << shift)) ==> orn
10590 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10591 iRegL src1, iRegL src2,
10592 immI src3, immL_M1 src4) %{
10593 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10594 ins_cost(1.9 * INSN_COST);
10595 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10596
10597 ins_encode %{
10598 __ orn(as_Register($dst$$reg),
10599 as_Register($src1$$reg),
10600 as_Register($src2$$reg),
10601 Assembler::LSL,
10602 $src3$$constant & 0x3f);
10603 %}
10604
10605 ins_pipe(ialu_reg_reg_shift);
10606 %}
10607
10608 // This pattern is automatically generated from aarch64_ad.m4
10609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10610 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10611 iRegIorL2I src1, iRegIorL2I src2,
10612 immI src3) %{
10613 match(Set dst (AndI src1 (URShiftI src2 src3)));
10614
10615 ins_cost(1.9 * INSN_COST);
10616 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10617
10618 ins_encode %{
10619 __ andw(as_Register($dst$$reg),
10620 as_Register($src1$$reg),
10621 as_Register($src2$$reg),
10622 Assembler::LSR,
10623 $src3$$constant & 0x1f);
10624 %}
10625
10626 ins_pipe(ialu_reg_reg_shift);
10627 %}
10628
10629 // This pattern is automatically generated from aarch64_ad.m4
10630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10631 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10632 iRegL src1, iRegL src2,
10633 immI src3) %{
10634 match(Set dst (AndL src1 (URShiftL src2 src3)));
10635
10636 ins_cost(1.9 * INSN_COST);
10637 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10638
10639 ins_encode %{
10640 __ andr(as_Register($dst$$reg),
10641 as_Register($src1$$reg),
10642 as_Register($src2$$reg),
10643 Assembler::LSR,
10644 $src3$$constant & 0x3f);
10645 %}
10646
10647 ins_pipe(ialu_reg_reg_shift);
10648 %}
10649
10650 // This pattern is automatically generated from aarch64_ad.m4
10651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10652 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10653 iRegIorL2I src1, iRegIorL2I src2,
10654 immI src3) %{
10655 match(Set dst (AndI src1 (RShiftI src2 src3)));
10656
10657 ins_cost(1.9 * INSN_COST);
10658 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10659
10660 ins_encode %{
10661 __ andw(as_Register($dst$$reg),
10662 as_Register($src1$$reg),
10663 as_Register($src2$$reg),
10664 Assembler::ASR,
10665 $src3$$constant & 0x1f);
10666 %}
10667
10668 ins_pipe(ialu_reg_reg_shift);
10669 %}
10670
10671 // This pattern is automatically generated from aarch64_ad.m4
10672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10673 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10674 iRegL src1, iRegL src2,
10675 immI src3) %{
10676 match(Set dst (AndL src1 (RShiftL src2 src3)));
10677
10678 ins_cost(1.9 * INSN_COST);
10679 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10680
10681 ins_encode %{
10682 __ andr(as_Register($dst$$reg),
10683 as_Register($src1$$reg),
10684 as_Register($src2$$reg),
10685 Assembler::ASR,
10686 $src3$$constant & 0x3f);
10687 %}
10688
10689 ins_pipe(ialu_reg_reg_shift);
10690 %}
10691
10692 // This pattern is automatically generated from aarch64_ad.m4
10693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10694 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10695 iRegIorL2I src1, iRegIorL2I src2,
10696 immI src3) %{
10697 match(Set dst (AndI src1 (LShiftI src2 src3)));
10698
10699 ins_cost(1.9 * INSN_COST);
10700 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10701
10702 ins_encode %{
10703 __ andw(as_Register($dst$$reg),
10704 as_Register($src1$$reg),
10705 as_Register($src2$$reg),
10706 Assembler::LSL,
10707 $src3$$constant & 0x1f);
10708 %}
10709
10710 ins_pipe(ialu_reg_reg_shift);
10711 %}
10712
10713 // This pattern is automatically generated from aarch64_ad.m4
10714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10715 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10716 iRegL src1, iRegL src2,
10717 immI src3) %{
10718 match(Set dst (AndL src1 (LShiftL src2 src3)));
10719
10720 ins_cost(1.9 * INSN_COST);
10721 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10722
10723 ins_encode %{
10724 __ andr(as_Register($dst$$reg),
10725 as_Register($src1$$reg),
10726 as_Register($src2$$reg),
10727 Assembler::LSL,
10728 $src3$$constant & 0x3f);
10729 %}
10730
10731 ins_pipe(ialu_reg_reg_shift);
10732 %}
10733
10734 // This pattern is automatically generated from aarch64_ad.m4
10735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10736 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10737 iRegIorL2I src1, iRegIorL2I src2,
10738 immI src3) %{
10739 match(Set dst (AndI src1 (RotateRight src2 src3)));
10740
10741 ins_cost(1.9 * INSN_COST);
10742 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10743
10744 ins_encode %{
10745 __ andw(as_Register($dst$$reg),
10746 as_Register($src1$$reg),
10747 as_Register($src2$$reg),
10748 Assembler::ROR,
10749 $src3$$constant & 0x1f);
10750 %}
10751
10752 ins_pipe(ialu_reg_reg_shift);
10753 %}
10754
10755 // This pattern is automatically generated from aarch64_ad.m4
10756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10757 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10758 iRegL src1, iRegL src2,
10759 immI src3) %{
10760 match(Set dst (AndL src1 (RotateRight src2 src3)));
10761
10762 ins_cost(1.9 * INSN_COST);
10763 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10764
10765 ins_encode %{
10766 __ andr(as_Register($dst$$reg),
10767 as_Register($src1$$reg),
10768 as_Register($src2$$reg),
10769 Assembler::ROR,
10770 $src3$$constant & 0x3f);
10771 %}
10772
10773 ins_pipe(ialu_reg_reg_shift);
10774 %}
10775
10776 // This pattern is automatically generated from aarch64_ad.m4
10777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10778 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10779 iRegIorL2I src1, iRegIorL2I src2,
10780 immI src3) %{
10781 match(Set dst (XorI src1 (URShiftI src2 src3)));
10782
10783 ins_cost(1.9 * INSN_COST);
10784 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10785
10786 ins_encode %{
10787 __ eorw(as_Register($dst$$reg),
10788 as_Register($src1$$reg),
10789 as_Register($src2$$reg),
10790 Assembler::LSR,
10791 $src3$$constant & 0x1f);
10792 %}
10793
10794 ins_pipe(ialu_reg_reg_shift);
10795 %}
10796
10797 // This pattern is automatically generated from aarch64_ad.m4
10798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10799 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10800 iRegL src1, iRegL src2,
10801 immI src3) %{
10802 match(Set dst (XorL src1 (URShiftL src2 src3)));
10803
10804 ins_cost(1.9 * INSN_COST);
10805 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10806
10807 ins_encode %{
10808 __ eor(as_Register($dst$$reg),
10809 as_Register($src1$$reg),
10810 as_Register($src2$$reg),
10811 Assembler::LSR,
10812 $src3$$constant & 0x3f);
10813 %}
10814
10815 ins_pipe(ialu_reg_reg_shift);
10816 %}
10817
10818 // This pattern is automatically generated from aarch64_ad.m4
10819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10820 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10821 iRegIorL2I src1, iRegIorL2I src2,
10822 immI src3) %{
10823 match(Set dst (XorI src1 (RShiftI src2 src3)));
10824
10825 ins_cost(1.9 * INSN_COST);
10826 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10827
10828 ins_encode %{
10829 __ eorw(as_Register($dst$$reg),
10830 as_Register($src1$$reg),
10831 as_Register($src2$$reg),
10832 Assembler::ASR,
10833 $src3$$constant & 0x1f);
10834 %}
10835
10836 ins_pipe(ialu_reg_reg_shift);
10837 %}
10838
10839 // This pattern is automatically generated from aarch64_ad.m4
10840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10841 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10842 iRegL src1, iRegL src2,
10843 immI src3) %{
10844 match(Set dst (XorL src1 (RShiftL src2 src3)));
10845
10846 ins_cost(1.9 * INSN_COST);
10847 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10848
10849 ins_encode %{
10850 __ eor(as_Register($dst$$reg),
10851 as_Register($src1$$reg),
10852 as_Register($src2$$reg),
10853 Assembler::ASR,
10854 $src3$$constant & 0x3f);
10855 %}
10856
10857 ins_pipe(ialu_reg_reg_shift);
10858 %}
10859
10860 // This pattern is automatically generated from aarch64_ad.m4
10861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10862 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10863 iRegIorL2I src1, iRegIorL2I src2,
10864 immI src3) %{
10865 match(Set dst (XorI src1 (LShiftI src2 src3)));
10866
10867 ins_cost(1.9 * INSN_COST);
10868 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10869
10870 ins_encode %{
10871 __ eorw(as_Register($dst$$reg),
10872 as_Register($src1$$reg),
10873 as_Register($src2$$reg),
10874 Assembler::LSL,
10875 $src3$$constant & 0x1f);
10876 %}
10877
10878 ins_pipe(ialu_reg_reg_shift);
10879 %}
10880
10881 // This pattern is automatically generated from aarch64_ad.m4
10882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10883 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10884 iRegL src1, iRegL src2,
10885 immI src3) %{
10886 match(Set dst (XorL src1 (LShiftL src2 src3)));
10887
10888 ins_cost(1.9 * INSN_COST);
10889 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10890
10891 ins_encode %{
10892 __ eor(as_Register($dst$$reg),
10893 as_Register($src1$$reg),
10894 as_Register($src2$$reg),
10895 Assembler::LSL,
10896 $src3$$constant & 0x3f);
10897 %}
10898
10899 ins_pipe(ialu_reg_reg_shift);
10900 %}
10901
10902 // This pattern is automatically generated from aarch64_ad.m4
10903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10904 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10905 iRegIorL2I src1, iRegIorL2I src2,
10906 immI src3) %{
10907 match(Set dst (XorI src1 (RotateRight src2 src3)));
10908
10909 ins_cost(1.9 * INSN_COST);
10910 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10911
10912 ins_encode %{
10913 __ eorw(as_Register($dst$$reg),
10914 as_Register($src1$$reg),
10915 as_Register($src2$$reg),
10916 Assembler::ROR,
10917 $src3$$constant & 0x1f);
10918 %}
10919
10920 ins_pipe(ialu_reg_reg_shift);
10921 %}
10922
10923 // This pattern is automatically generated from aarch64_ad.m4
10924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10925 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10926 iRegL src1, iRegL src2,
10927 immI src3) %{
10928 match(Set dst (XorL src1 (RotateRight src2 src3)));
10929
10930 ins_cost(1.9 * INSN_COST);
10931 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10932
10933 ins_encode %{
10934 __ eor(as_Register($dst$$reg),
10935 as_Register($src1$$reg),
10936 as_Register($src2$$reg),
10937 Assembler::ROR,
10938 $src3$$constant & 0x3f);
10939 %}
10940
10941 ins_pipe(ialu_reg_reg_shift);
10942 %}
10943
10944 // This pattern is automatically generated from aarch64_ad.m4
10945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10946 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10947 iRegIorL2I src1, iRegIorL2I src2,
10948 immI src3) %{
10949 match(Set dst (OrI src1 (URShiftI src2 src3)));
10950
10951 ins_cost(1.9 * INSN_COST);
10952 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10953
10954 ins_encode %{
10955 __ orrw(as_Register($dst$$reg),
10956 as_Register($src1$$reg),
10957 as_Register($src2$$reg),
10958 Assembler::LSR,
10959 $src3$$constant & 0x1f);
10960 %}
10961
10962 ins_pipe(ialu_reg_reg_shift);
10963 %}
10964
10965 // This pattern is automatically generated from aarch64_ad.m4
10966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10967 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10968 iRegL src1, iRegL src2,
10969 immI src3) %{
10970 match(Set dst (OrL src1 (URShiftL src2 src3)));
10971
10972 ins_cost(1.9 * INSN_COST);
10973 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10974
10975 ins_encode %{
10976 __ orr(as_Register($dst$$reg),
10977 as_Register($src1$$reg),
10978 as_Register($src2$$reg),
10979 Assembler::LSR,
10980 $src3$$constant & 0x3f);
10981 %}
10982
10983 ins_pipe(ialu_reg_reg_shift);
10984 %}
10985
10986 // This pattern is automatically generated from aarch64_ad.m4
10987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10988 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10989 iRegIorL2I src1, iRegIorL2I src2,
10990 immI src3) %{
10991 match(Set dst (OrI src1 (RShiftI src2 src3)));
10992
10993 ins_cost(1.9 * INSN_COST);
10994 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10995
10996 ins_encode %{
10997 __ orrw(as_Register($dst$$reg),
10998 as_Register($src1$$reg),
10999 as_Register($src2$$reg),
11000 Assembler::ASR,
11001 $src3$$constant & 0x1f);
11002 %}
11003
11004 ins_pipe(ialu_reg_reg_shift);
11005 %}
11006
11007 // This pattern is automatically generated from aarch64_ad.m4
11008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11009 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11010 iRegL src1, iRegL src2,
11011 immI src3) %{
11012 match(Set dst (OrL src1 (RShiftL src2 src3)));
11013
11014 ins_cost(1.9 * INSN_COST);
11015 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11016
11017 ins_encode %{
11018 __ orr(as_Register($dst$$reg),
11019 as_Register($src1$$reg),
11020 as_Register($src2$$reg),
11021 Assembler::ASR,
11022 $src3$$constant & 0x3f);
11023 %}
11024
11025 ins_pipe(ialu_reg_reg_shift);
11026 %}
11027
11028 // This pattern is automatically generated from aarch64_ad.m4
11029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11030 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11031 iRegIorL2I src1, iRegIorL2I src2,
11032 immI src3) %{
11033 match(Set dst (OrI src1 (LShiftI src2 src3)));
11034
11035 ins_cost(1.9 * INSN_COST);
11036 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11037
11038 ins_encode %{
11039 __ orrw(as_Register($dst$$reg),
11040 as_Register($src1$$reg),
11041 as_Register($src2$$reg),
11042 Assembler::LSL,
11043 $src3$$constant & 0x1f);
11044 %}
11045
11046 ins_pipe(ialu_reg_reg_shift);
11047 %}
11048
11049 // This pattern is automatically generated from aarch64_ad.m4
11050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11051 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11052 iRegL src1, iRegL src2,
11053 immI src3) %{
11054 match(Set dst (OrL src1 (LShiftL src2 src3)));
11055
11056 ins_cost(1.9 * INSN_COST);
11057 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11058
11059 ins_encode %{
11060 __ orr(as_Register($dst$$reg),
11061 as_Register($src1$$reg),
11062 as_Register($src2$$reg),
11063 Assembler::LSL,
11064 $src3$$constant & 0x3f);
11065 %}
11066
11067 ins_pipe(ialu_reg_reg_shift);
11068 %}
11069
11070 // This pattern is automatically generated from aarch64_ad.m4
11071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11072 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11073 iRegIorL2I src1, iRegIorL2I src2,
11074 immI src3) %{
11075 match(Set dst (OrI src1 (RotateRight src2 src3)));
11076
11077 ins_cost(1.9 * INSN_COST);
11078 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11079
11080 ins_encode %{
11081 __ orrw(as_Register($dst$$reg),
11082 as_Register($src1$$reg),
11083 as_Register($src2$$reg),
11084 Assembler::ROR,
11085 $src3$$constant & 0x1f);
11086 %}
11087
11088 ins_pipe(ialu_reg_reg_shift);
11089 %}
11090
11091 // This pattern is automatically generated from aarch64_ad.m4
11092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11093 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11094 iRegL src1, iRegL src2,
11095 immI src3) %{
11096 match(Set dst (OrL src1 (RotateRight src2 src3)));
11097
11098 ins_cost(1.9 * INSN_COST);
11099 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11100
11101 ins_encode %{
11102 __ orr(as_Register($dst$$reg),
11103 as_Register($src1$$reg),
11104 as_Register($src2$$reg),
11105 Assembler::ROR,
11106 $src3$$constant & 0x3f);
11107 %}
11108
11109 ins_pipe(ialu_reg_reg_shift);
11110 %}
11111
11112 // This pattern is automatically generated from aarch64_ad.m4
11113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11114 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11115 iRegIorL2I src1, iRegIorL2I src2,
11116 immI src3) %{
11117 match(Set dst (AddI src1 (URShiftI src2 src3)));
11118
11119 ins_cost(1.9 * INSN_COST);
11120 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11121
11122 ins_encode %{
11123 __ addw(as_Register($dst$$reg),
11124 as_Register($src1$$reg),
11125 as_Register($src2$$reg),
11126 Assembler::LSR,
11127 $src3$$constant & 0x1f);
11128 %}
11129
11130 ins_pipe(ialu_reg_reg_shift);
11131 %}
11132
11133 // This pattern is automatically generated from aarch64_ad.m4
11134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11135 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11136 iRegL src1, iRegL src2,
11137 immI src3) %{
11138 match(Set dst (AddL src1 (URShiftL src2 src3)));
11139
11140 ins_cost(1.9 * INSN_COST);
11141 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11142
11143 ins_encode %{
11144 __ add(as_Register($dst$$reg),
11145 as_Register($src1$$reg),
11146 as_Register($src2$$reg),
11147 Assembler::LSR,
11148 $src3$$constant & 0x3f);
11149 %}
11150
11151 ins_pipe(ialu_reg_reg_shift);
11152 %}
11153
11154 // This pattern is automatically generated from aarch64_ad.m4
11155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11156 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11157 iRegIorL2I src1, iRegIorL2I src2,
11158 immI src3) %{
11159 match(Set dst (AddI src1 (RShiftI src2 src3)));
11160
11161 ins_cost(1.9 * INSN_COST);
11162 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11163
11164 ins_encode %{
11165 __ addw(as_Register($dst$$reg),
11166 as_Register($src1$$reg),
11167 as_Register($src2$$reg),
11168 Assembler::ASR,
11169 $src3$$constant & 0x1f);
11170 %}
11171
11172 ins_pipe(ialu_reg_reg_shift);
11173 %}
11174
11175 // This pattern is automatically generated from aarch64_ad.m4
11176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11177 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11178 iRegL src1, iRegL src2,
11179 immI src3) %{
11180 match(Set dst (AddL src1 (RShiftL src2 src3)));
11181
11182 ins_cost(1.9 * INSN_COST);
11183 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11184
11185 ins_encode %{
11186 __ add(as_Register($dst$$reg),
11187 as_Register($src1$$reg),
11188 as_Register($src2$$reg),
11189 Assembler::ASR,
11190 $src3$$constant & 0x3f);
11191 %}
11192
11193 ins_pipe(ialu_reg_reg_shift);
11194 %}
11195
11196 // This pattern is automatically generated from aarch64_ad.m4
11197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11198 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11199 iRegIorL2I src1, iRegIorL2I src2,
11200 immI src3) %{
11201 match(Set dst (AddI src1 (LShiftI src2 src3)));
11202
11203 ins_cost(1.9 * INSN_COST);
11204 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11205
11206 ins_encode %{
11207 __ addw(as_Register($dst$$reg),
11208 as_Register($src1$$reg),
11209 as_Register($src2$$reg),
11210 Assembler::LSL,
11211 $src3$$constant & 0x1f);
11212 %}
11213
11214 ins_pipe(ialu_reg_reg_shift);
11215 %}
11216
11217 // This pattern is automatically generated from aarch64_ad.m4
11218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11219 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11220 iRegL src1, iRegL src2,
11221 immI src3) %{
11222 match(Set dst (AddL src1 (LShiftL src2 src3)));
11223
11224 ins_cost(1.9 * INSN_COST);
11225 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11226
11227 ins_encode %{
11228 __ add(as_Register($dst$$reg),
11229 as_Register($src1$$reg),
11230 as_Register($src2$$reg),
11231 Assembler::LSL,
11232 $src3$$constant & 0x3f);
11233 %}
11234
11235 ins_pipe(ialu_reg_reg_shift);
11236 %}
11237
11238 // This pattern is automatically generated from aarch64_ad.m4
11239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11240 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11241 iRegIorL2I src1, iRegIorL2I src2,
11242 immI src3) %{
11243 match(Set dst (SubI src1 (URShiftI src2 src3)));
11244
11245 ins_cost(1.9 * INSN_COST);
11246 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11247
11248 ins_encode %{
11249 __ subw(as_Register($dst$$reg),
11250 as_Register($src1$$reg),
11251 as_Register($src2$$reg),
11252 Assembler::LSR,
11253 $src3$$constant & 0x1f);
11254 %}
11255
11256 ins_pipe(ialu_reg_reg_shift);
11257 %}
11258
11259 // This pattern is automatically generated from aarch64_ad.m4
11260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11261 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11262 iRegL src1, iRegL src2,
11263 immI src3) %{
11264 match(Set dst (SubL src1 (URShiftL src2 src3)));
11265
11266 ins_cost(1.9 * INSN_COST);
11267 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11268
11269 ins_encode %{
11270 __ sub(as_Register($dst$$reg),
11271 as_Register($src1$$reg),
11272 as_Register($src2$$reg),
11273 Assembler::LSR,
11274 $src3$$constant & 0x3f);
11275 %}
11276
11277 ins_pipe(ialu_reg_reg_shift);
11278 %}
11279
11280 // This pattern is automatically generated from aarch64_ad.m4
11281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11282 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11283 iRegIorL2I src1, iRegIorL2I src2,
11284 immI src3) %{
11285 match(Set dst (SubI src1 (RShiftI src2 src3)));
11286
11287 ins_cost(1.9 * INSN_COST);
11288 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11289
11290 ins_encode %{
11291 __ subw(as_Register($dst$$reg),
11292 as_Register($src1$$reg),
11293 as_Register($src2$$reg),
11294 Assembler::ASR,
11295 $src3$$constant & 0x1f);
11296 %}
11297
11298 ins_pipe(ialu_reg_reg_shift);
11299 %}
11300
11301 // This pattern is automatically generated from aarch64_ad.m4
11302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11303 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11304 iRegL src1, iRegL src2,
11305 immI src3) %{
11306 match(Set dst (SubL src1 (RShiftL src2 src3)));
11307
11308 ins_cost(1.9 * INSN_COST);
11309 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11310
11311 ins_encode %{
11312 __ sub(as_Register($dst$$reg),
11313 as_Register($src1$$reg),
11314 as_Register($src2$$reg),
11315 Assembler::ASR,
11316 $src3$$constant & 0x3f);
11317 %}
11318
11319 ins_pipe(ialu_reg_reg_shift);
11320 %}
11321
11322 // This pattern is automatically generated from aarch64_ad.m4
11323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11324 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11325 iRegIorL2I src1, iRegIorL2I src2,
11326 immI src3) %{
11327 match(Set dst (SubI src1 (LShiftI src2 src3)));
11328
11329 ins_cost(1.9 * INSN_COST);
11330 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11331
11332 ins_encode %{
11333 __ subw(as_Register($dst$$reg),
11334 as_Register($src1$$reg),
11335 as_Register($src2$$reg),
11336 Assembler::LSL,
11337 $src3$$constant & 0x1f);
11338 %}
11339
11340 ins_pipe(ialu_reg_reg_shift);
11341 %}
11342
11343 // This pattern is automatically generated from aarch64_ad.m4
11344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11345 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11346 iRegL src1, iRegL src2,
11347 immI src3) %{
11348 match(Set dst (SubL src1 (LShiftL src2 src3)));
11349
11350 ins_cost(1.9 * INSN_COST);
11351 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11352
11353 ins_encode %{
11354 __ sub(as_Register($dst$$reg),
11355 as_Register($src1$$reg),
11356 as_Register($src2$$reg),
11357 Assembler::LSL,
11358 $src3$$constant & 0x3f);
11359 %}
11360
11361 ins_pipe(ialu_reg_reg_shift);
11362 %}
11363
11364 // This pattern is automatically generated from aarch64_ad.m4
11365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11366
11367 // Shift Left followed by Shift Right.
11368 // This idiom is used by the compiler for the i2b bytecode etc.
11369 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11370 %{
11371 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11372 ins_cost(INSN_COST * 2);
11373 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11374 ins_encode %{
11375 int lshift = $lshift_count$$constant & 63;
11376 int rshift = $rshift_count$$constant & 63;
11377 int s = 63 - lshift;
11378 int r = (rshift - lshift) & 63;
11379 __ sbfm(as_Register($dst$$reg),
11380 as_Register($src$$reg),
11381 r, s);
11382 %}
11383
11384 ins_pipe(ialu_reg_shift);
11385 %}
11386
11387 // This pattern is automatically generated from aarch64_ad.m4
11388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11389
11390 // Shift Left followed by Shift Right.
11391 // This idiom is used by the compiler for the i2b bytecode etc.
11392 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11393 %{
11394 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11395 ins_cost(INSN_COST * 2);
11396 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11397 ins_encode %{
11398 int lshift = $lshift_count$$constant & 31;
11399 int rshift = $rshift_count$$constant & 31;
11400 int s = 31 - lshift;
11401 int r = (rshift - lshift) & 31;
11402 __ sbfmw(as_Register($dst$$reg),
11403 as_Register($src$$reg),
11404 r, s);
11405 %}
11406
11407 ins_pipe(ialu_reg_shift);
11408 %}
11409
11410 // This pattern is automatically generated from aarch64_ad.m4
11411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11412
11413 // Shift Left followed by Shift Right.
11414 // This idiom is used by the compiler for the i2b bytecode etc.
11415 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11416 %{
11417 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11418 ins_cost(INSN_COST * 2);
11419 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11420 ins_encode %{
11421 int lshift = $lshift_count$$constant & 63;
11422 int rshift = $rshift_count$$constant & 63;
11423 int s = 63 - lshift;
11424 int r = (rshift - lshift) & 63;
11425 __ ubfm(as_Register($dst$$reg),
11426 as_Register($src$$reg),
11427 r, s);
11428 %}
11429
11430 ins_pipe(ialu_reg_shift);
11431 %}
11432
11433 // This pattern is automatically generated from aarch64_ad.m4
11434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11435
11436 // Shift Left followed by Shift Right.
11437 // This idiom is used by the compiler for the i2b bytecode etc.
11438 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11439 %{
11440 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11441 ins_cost(INSN_COST * 2);
11442 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11443 ins_encode %{
11444 int lshift = $lshift_count$$constant & 31;
11445 int rshift = $rshift_count$$constant & 31;
11446 int s = 31 - lshift;
11447 int r = (rshift - lshift) & 31;
11448 __ ubfmw(as_Register($dst$$reg),
11449 as_Register($src$$reg),
11450 r, s);
11451 %}
11452
11453 ins_pipe(ialu_reg_shift);
11454 %}
11455
11456 // Bitfield extract with shift & mask
11457
11458 // This pattern is automatically generated from aarch64_ad.m4
11459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11460 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11461 %{
11462 match(Set dst (AndI (URShiftI src rshift) mask));
11463 // Make sure we are not going to exceed what ubfxw can do.
11464 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11465
11466 ins_cost(INSN_COST);
11467 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11468 ins_encode %{
11469 int rshift = $rshift$$constant & 31;
11470 intptr_t mask = $mask$$constant;
11471 int width = exact_log2(mask+1);
11472 __ ubfxw(as_Register($dst$$reg),
11473 as_Register($src$$reg), rshift, width);
11474 %}
11475 ins_pipe(ialu_reg_shift);
11476 %}
11477
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11481 %{
11482 match(Set dst (AndL (URShiftL src rshift) mask));
11483 // Make sure we are not going to exceed what ubfx can do.
11484 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11485
11486 ins_cost(INSN_COST);
11487 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11488 ins_encode %{
11489 int rshift = $rshift$$constant & 63;
11490 intptr_t mask = $mask$$constant;
11491 int width = exact_log2_long(mask+1);
11492 __ ubfx(as_Register($dst$$reg),
11493 as_Register($src$$reg), rshift, width);
11494 %}
11495 ins_pipe(ialu_reg_shift);
11496 %}
11497
11498
11499 // This pattern is automatically generated from aarch64_ad.m4
11500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11501
11502 // We can use ubfx when extending an And with a mask when we know mask
11503 // is positive. We know that because immI_bitmask guarantees it.
11504 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11505 %{
11506 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11507 // Make sure we are not going to exceed what ubfxw can do.
11508 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11509
11510 ins_cost(INSN_COST * 2);
11511 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11512 ins_encode %{
11513 int rshift = $rshift$$constant & 31;
11514 intptr_t mask = $mask$$constant;
11515 int width = exact_log2(mask+1);
11516 __ ubfx(as_Register($dst$$reg),
11517 as_Register($src$$reg), rshift, width);
11518 %}
11519 ins_pipe(ialu_reg_shift);
11520 %}
11521
11522
11523 // This pattern is automatically generated from aarch64_ad.m4
11524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11525
11526 // We can use ubfiz when masking by a positive number and then left shifting the result.
11527 // We know that the mask is positive because immI_bitmask guarantees it.
11528 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11529 %{
11530 match(Set dst (LShiftI (AndI src mask) lshift));
11531 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11532
11533 ins_cost(INSN_COST);
11534 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11535 ins_encode %{
11536 int lshift = $lshift$$constant & 31;
11537 intptr_t mask = $mask$$constant;
11538 int width = exact_log2(mask+1);
11539 __ ubfizw(as_Register($dst$$reg),
11540 as_Register($src$$reg), lshift, width);
11541 %}
11542 ins_pipe(ialu_reg_shift);
11543 %}
11544
11545 // This pattern is automatically generated from aarch64_ad.m4
11546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11547
11548 // We can use ubfiz when masking by a positive number and then left shifting the result.
11549 // We know that the mask is positive because immL_bitmask guarantees it.
11550 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11551 %{
11552 match(Set dst (LShiftL (AndL src mask) lshift));
11553 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11554
11555 ins_cost(INSN_COST);
11556 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11557 ins_encode %{
11558 int lshift = $lshift$$constant & 63;
11559 intptr_t mask = $mask$$constant;
11560 int width = exact_log2_long(mask+1);
11561 __ ubfiz(as_Register($dst$$reg),
11562 as_Register($src$$reg), lshift, width);
11563 %}
11564 ins_pipe(ialu_reg_shift);
11565 %}
11566
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569
11570 // We can use ubfiz when masking by a positive number and then left shifting the result.
11571 // We know that the mask is positive because immI_bitmask guarantees it.
11572 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11573 %{
11574 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11575 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11576
11577 ins_cost(INSN_COST);
11578 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11579 ins_encode %{
11580 int lshift = $lshift$$constant & 31;
11581 intptr_t mask = $mask$$constant;
11582 int width = exact_log2(mask+1);
11583 __ ubfizw(as_Register($dst$$reg),
11584 as_Register($src$$reg), lshift, width);
11585 %}
11586 ins_pipe(ialu_reg_shift);
11587 %}
11588
11589 // This pattern is automatically generated from aarch64_ad.m4
11590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11591
11592 // We can use ubfiz when masking by a positive number and then left shifting the result.
11593 // We know that the mask is positive because immL_bitmask guarantees it.
11594 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11595 %{
11596 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11597 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11598
11599 ins_cost(INSN_COST);
11600 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11601 ins_encode %{
11602 int lshift = $lshift$$constant & 63;
11603 intptr_t mask = $mask$$constant;
11604 int width = exact_log2_long(mask+1);
11605 __ ubfiz(as_Register($dst$$reg),
11606 as_Register($src$$reg), lshift, width);
11607 %}
11608 ins_pipe(ialu_reg_shift);
11609 %}
11610
11611
11612 // This pattern is automatically generated from aarch64_ad.m4
11613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11614
11615 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11616 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11617 %{
11618 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11619 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11620
11621 ins_cost(INSN_COST);
11622 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11623 ins_encode %{
11624 int lshift = $lshift$$constant & 63;
11625 intptr_t mask = $mask$$constant;
11626 int width = exact_log2(mask+1);
11627 __ ubfiz(as_Register($dst$$reg),
11628 as_Register($src$$reg), lshift, width);
11629 %}
11630 ins_pipe(ialu_reg_shift);
11631 %}
11632
11633 // This pattern is automatically generated from aarch64_ad.m4
11634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11635
11636 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11637 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11638 %{
11639 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11640 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11641
11642 ins_cost(INSN_COST);
11643 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11644 ins_encode %{
11645 int lshift = $lshift$$constant & 31;
11646 intptr_t mask = $mask$$constant;
11647 int width = exact_log2(mask+1);
11648 __ ubfiz(as_Register($dst$$reg),
11649 as_Register($src$$reg), lshift, width);
11650 %}
11651 ins_pipe(ialu_reg_shift);
11652 %}
11653
11654 // This pattern is automatically generated from aarch64_ad.m4
11655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11656
11657 // Can skip int2long conversions after AND with small bitmask
11658 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11659 %{
11660 match(Set dst (ConvI2L (AndI src msk)));
11661 ins_cost(INSN_COST);
11662 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11663 ins_encode %{
11664 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11665 %}
11666 ins_pipe(ialu_reg_shift);
11667 %}
11668
11669
11670 // Rotations
11671
11672 // This pattern is automatically generated from aarch64_ad.m4
11673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11674 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11675 %{
11676 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11677 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11678
11679 ins_cost(INSN_COST);
11680 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11681
11682 ins_encode %{
11683 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11684 $rshift$$constant & 63);
11685 %}
11686 ins_pipe(ialu_reg_reg_extr);
11687 %}
11688
11689
11690 // This pattern is automatically generated from aarch64_ad.m4
11691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11692 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11693 %{
11694 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11695 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11696
11697 ins_cost(INSN_COST);
11698 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11699
11700 ins_encode %{
11701 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11702 $rshift$$constant & 31);
11703 %}
11704 ins_pipe(ialu_reg_reg_extr);
11705 %}
11706
11707
11708 // This pattern is automatically generated from aarch64_ad.m4
11709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11710 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11711 %{
11712 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11713 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11714
11715 ins_cost(INSN_COST);
11716 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11717
11718 ins_encode %{
11719 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11720 $rshift$$constant & 63);
11721 %}
11722 ins_pipe(ialu_reg_reg_extr);
11723 %}
11724
11725
11726 // This pattern is automatically generated from aarch64_ad.m4
11727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11728 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11729 %{
11730 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11731 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11732
11733 ins_cost(INSN_COST);
11734 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11735
11736 ins_encode %{
11737 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11738 $rshift$$constant & 31);
11739 %}
11740 ins_pipe(ialu_reg_reg_extr);
11741 %}
11742
11743 // This pattern is automatically generated from aarch64_ad.m4
11744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11745 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11746 %{
11747 match(Set dst (RotateRight src shift));
11748
11749 ins_cost(INSN_COST);
11750 format %{ "ror $dst, $src, $shift" %}
11751
11752 ins_encode %{
11753 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11754 $shift$$constant & 0x1f);
11755 %}
11756 ins_pipe(ialu_reg_reg_vshift);
11757 %}
11758
11759 // This pattern is automatically generated from aarch64_ad.m4
11760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11761 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11762 %{
11763 match(Set dst (RotateRight src shift));
11764
11765 ins_cost(INSN_COST);
11766 format %{ "ror $dst, $src, $shift" %}
11767
11768 ins_encode %{
11769 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11770 $shift$$constant & 0x3f);
11771 %}
11772 ins_pipe(ialu_reg_reg_vshift);
11773 %}
11774
11775 // This pattern is automatically generated from aarch64_ad.m4
11776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11777 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11778 %{
11779 match(Set dst (RotateRight src shift));
11780
11781 ins_cost(INSN_COST);
11782 format %{ "ror $dst, $src, $shift" %}
11783
11784 ins_encode %{
11785 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11786 %}
11787 ins_pipe(ialu_reg_reg_vshift);
11788 %}
11789
11790 // This pattern is automatically generated from aarch64_ad.m4
11791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11792 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11793 %{
11794 match(Set dst (RotateRight src shift));
11795
11796 ins_cost(INSN_COST);
11797 format %{ "ror $dst, $src, $shift" %}
11798
11799 ins_encode %{
11800 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11801 %}
11802 ins_pipe(ialu_reg_reg_vshift);
11803 %}
11804
11805 // This pattern is automatically generated from aarch64_ad.m4
11806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11807 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11808 %{
11809 match(Set dst (RotateLeft src shift));
11810
11811 ins_cost(INSN_COST);
11812 format %{ "rol $dst, $src, $shift" %}
11813
11814 ins_encode %{
11815 __ subw(rscratch1, zr, as_Register($shift$$reg));
11816 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11817 %}
11818 ins_pipe(ialu_reg_reg_vshift);
11819 %}
11820
11821 // This pattern is automatically generated from aarch64_ad.m4
11822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11823 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11824 %{
11825 match(Set dst (RotateLeft src shift));
11826
11827 ins_cost(INSN_COST);
11828 format %{ "rol $dst, $src, $shift" %}
11829
11830 ins_encode %{
11831 __ subw(rscratch1, zr, as_Register($shift$$reg));
11832 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11833 %}
11834 ins_pipe(ialu_reg_reg_vshift);
11835 %}
11836
11837
11838 // Add/subtract (extended)
11839
11840 // This pattern is automatically generated from aarch64_ad.m4
11841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11842 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11843 %{
11844 match(Set dst (AddL src1 (ConvI2L src2)));
11845 ins_cost(INSN_COST);
11846 format %{ "add $dst, $src1, $src2, sxtw" %}
11847
11848 ins_encode %{
11849 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11850 as_Register($src2$$reg), ext::sxtw);
11851 %}
11852 ins_pipe(ialu_reg_reg);
11853 %}
11854
11855 // This pattern is automatically generated from aarch64_ad.m4
11856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11857 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11858 %{
11859 match(Set dst (SubL src1 (ConvI2L src2)));
11860 ins_cost(INSN_COST);
11861 format %{ "sub $dst, $src1, $src2, sxtw" %}
11862
11863 ins_encode %{
11864 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11865 as_Register($src2$$reg), ext::sxtw);
11866 %}
11867 ins_pipe(ialu_reg_reg);
11868 %}
11869
11870 // This pattern is automatically generated from aarch64_ad.m4
11871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11872 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11873 %{
11874 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11875 ins_cost(INSN_COST);
11876 format %{ "add $dst, $src1, $src2, sxth" %}
11877
11878 ins_encode %{
11879 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11880 as_Register($src2$$reg), ext::sxth);
11881 %}
11882 ins_pipe(ialu_reg_reg);
11883 %}
11884
11885 // This pattern is automatically generated from aarch64_ad.m4
11886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11887 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11888 %{
11889 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11890 ins_cost(INSN_COST);
11891 format %{ "add $dst, $src1, $src2, sxtb" %}
11892
11893 ins_encode %{
11894 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11895 as_Register($src2$$reg), ext::sxtb);
11896 %}
11897 ins_pipe(ialu_reg_reg);
11898 %}
11899
11900 // This pattern is automatically generated from aarch64_ad.m4
11901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11902 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11903 %{
11904 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11905 ins_cost(INSN_COST);
11906 format %{ "add $dst, $src1, $src2, uxtb" %}
11907
11908 ins_encode %{
11909 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11910 as_Register($src2$$reg), ext::uxtb);
11911 %}
11912 ins_pipe(ialu_reg_reg);
11913 %}
11914
11915 // This pattern is automatically generated from aarch64_ad.m4
11916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11917 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11918 %{
11919 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11920 ins_cost(INSN_COST);
11921 format %{ "add $dst, $src1, $src2, sxth" %}
11922
11923 ins_encode %{
11924 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11925 as_Register($src2$$reg), ext::sxth);
11926 %}
11927 ins_pipe(ialu_reg_reg);
11928 %}
11929
11930 // This pattern is automatically generated from aarch64_ad.m4
11931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11932 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11933 %{
11934 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11935 ins_cost(INSN_COST);
11936 format %{ "add $dst, $src1, $src2, sxtw" %}
11937
11938 ins_encode %{
11939 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11940 as_Register($src2$$reg), ext::sxtw);
11941 %}
11942 ins_pipe(ialu_reg_reg);
11943 %}
11944
11945 // This pattern is automatically generated from aarch64_ad.m4
11946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11947 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11948 %{
11949 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11950 ins_cost(INSN_COST);
11951 format %{ "add $dst, $src1, $src2, sxtb" %}
11952
11953 ins_encode %{
11954 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11955 as_Register($src2$$reg), ext::sxtb);
11956 %}
11957 ins_pipe(ialu_reg_reg);
11958 %}
11959
11960 // This pattern is automatically generated from aarch64_ad.m4
11961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11962 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11963 %{
11964 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11965 ins_cost(INSN_COST);
11966 format %{ "add $dst, $src1, $src2, uxtb" %}
11967
11968 ins_encode %{
11969 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11970 as_Register($src2$$reg), ext::uxtb);
11971 %}
11972 ins_pipe(ialu_reg_reg);
11973 %}
11974
11975 // This pattern is automatically generated from aarch64_ad.m4
11976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11977 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11978 %{
11979 match(Set dst (AddI src1 (AndI src2 mask)));
11980 ins_cost(INSN_COST);
11981 format %{ "addw $dst, $src1, $src2, uxtb" %}
11982
11983 ins_encode %{
11984 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11985 as_Register($src2$$reg), ext::uxtb);
11986 %}
11987 ins_pipe(ialu_reg_reg);
11988 %}
11989
11990 // This pattern is automatically generated from aarch64_ad.m4
11991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11992 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11993 %{
11994 match(Set dst (AddI src1 (AndI src2 mask)));
11995 ins_cost(INSN_COST);
11996 format %{ "addw $dst, $src1, $src2, uxth" %}
11997
11998 ins_encode %{
11999 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12000 as_Register($src2$$reg), ext::uxth);
12001 %}
12002 ins_pipe(ialu_reg_reg);
12003 %}
12004
12005 // This pattern is automatically generated from aarch64_ad.m4
12006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12007 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12008 %{
12009 match(Set dst (AddL src1 (AndL src2 mask)));
12010 ins_cost(INSN_COST);
12011 format %{ "add $dst, $src1, $src2, uxtb" %}
12012
12013 ins_encode %{
12014 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12015 as_Register($src2$$reg), ext::uxtb);
12016 %}
12017 ins_pipe(ialu_reg_reg);
12018 %}
12019
12020 // This pattern is automatically generated from aarch64_ad.m4
12021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12022 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12023 %{
12024 match(Set dst (AddL src1 (AndL src2 mask)));
12025 ins_cost(INSN_COST);
12026 format %{ "add $dst, $src1, $src2, uxth" %}
12027
12028 ins_encode %{
12029 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12030 as_Register($src2$$reg), ext::uxth);
12031 %}
12032 ins_pipe(ialu_reg_reg);
12033 %}
12034
12035 // This pattern is automatically generated from aarch64_ad.m4
12036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12037 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12038 %{
12039 match(Set dst (AddL src1 (AndL src2 mask)));
12040 ins_cost(INSN_COST);
12041 format %{ "add $dst, $src1, $src2, uxtw" %}
12042
12043 ins_encode %{
12044 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12045 as_Register($src2$$reg), ext::uxtw);
12046 %}
12047 ins_pipe(ialu_reg_reg);
12048 %}
12049
12050 // This pattern is automatically generated from aarch64_ad.m4
12051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12052 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12053 %{
12054 match(Set dst (SubI src1 (AndI src2 mask)));
12055 ins_cost(INSN_COST);
12056 format %{ "subw $dst, $src1, $src2, uxtb" %}
12057
12058 ins_encode %{
12059 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12060 as_Register($src2$$reg), ext::uxtb);
12061 %}
12062 ins_pipe(ialu_reg_reg);
12063 %}
12064
12065 // This pattern is automatically generated from aarch64_ad.m4
12066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12067 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12068 %{
12069 match(Set dst (SubI src1 (AndI src2 mask)));
12070 ins_cost(INSN_COST);
12071 format %{ "subw $dst, $src1, $src2, uxth" %}
12072
12073 ins_encode %{
12074 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12075 as_Register($src2$$reg), ext::uxth);
12076 %}
12077 ins_pipe(ialu_reg_reg);
12078 %}
12079
12080 // This pattern is automatically generated from aarch64_ad.m4
12081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12082 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12083 %{
12084 match(Set dst (SubL src1 (AndL src2 mask)));
12085 ins_cost(INSN_COST);
12086 format %{ "sub $dst, $src1, $src2, uxtb" %}
12087
12088 ins_encode %{
12089 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12090 as_Register($src2$$reg), ext::uxtb);
12091 %}
12092 ins_pipe(ialu_reg_reg);
12093 %}
12094
12095 // This pattern is automatically generated from aarch64_ad.m4
12096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12097 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12098 %{
12099 match(Set dst (SubL src1 (AndL src2 mask)));
12100 ins_cost(INSN_COST);
12101 format %{ "sub $dst, $src1, $src2, uxth" %}
12102
12103 ins_encode %{
12104 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12105 as_Register($src2$$reg), ext::uxth);
12106 %}
12107 ins_pipe(ialu_reg_reg);
12108 %}
12109
12110 // This pattern is automatically generated from aarch64_ad.m4
12111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12112 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12113 %{
12114 match(Set dst (SubL src1 (AndL src2 mask)));
12115 ins_cost(INSN_COST);
12116 format %{ "sub $dst, $src1, $src2, uxtw" %}
12117
12118 ins_encode %{
12119 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12120 as_Register($src2$$reg), ext::uxtw);
12121 %}
12122 ins_pipe(ialu_reg_reg);
12123 %}
12124
12125
12126 // This pattern is automatically generated from aarch64_ad.m4
12127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12128 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12129 %{
12130 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12131 ins_cost(1.9 * INSN_COST);
12132 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12133
12134 ins_encode %{
12135 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12136 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12137 %}
12138 ins_pipe(ialu_reg_reg_shift);
12139 %}
12140
12141 // This pattern is automatically generated from aarch64_ad.m4
12142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12143 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12144 %{
12145 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12146 ins_cost(1.9 * INSN_COST);
12147 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12148
12149 ins_encode %{
12150 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12151 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12152 %}
12153 ins_pipe(ialu_reg_reg_shift);
12154 %}
12155
12156 // This pattern is automatically generated from aarch64_ad.m4
12157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12158 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12159 %{
12160 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12161 ins_cost(1.9 * INSN_COST);
12162 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12163
12164 ins_encode %{
12165 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12166 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12167 %}
12168 ins_pipe(ialu_reg_reg_shift);
12169 %}
12170
12171 // This pattern is automatically generated from aarch64_ad.m4
12172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12173 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12174 %{
12175 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12176 ins_cost(1.9 * INSN_COST);
12177 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12178
12179 ins_encode %{
12180 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12181 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12182 %}
12183 ins_pipe(ialu_reg_reg_shift);
12184 %}
12185
12186 // This pattern is automatically generated from aarch64_ad.m4
12187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12188 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12189 %{
12190 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12191 ins_cost(1.9 * INSN_COST);
12192 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12193
12194 ins_encode %{
12195 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12196 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12197 %}
12198 ins_pipe(ialu_reg_reg_shift);
12199 %}
12200
12201 // This pattern is automatically generated from aarch64_ad.m4
12202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12203 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12204 %{
12205 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12206 ins_cost(1.9 * INSN_COST);
12207 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12208
12209 ins_encode %{
12210 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12211 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12212 %}
12213 ins_pipe(ialu_reg_reg_shift);
12214 %}
12215
12216 // This pattern is automatically generated from aarch64_ad.m4
12217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12218 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12219 %{
12220 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12221 ins_cost(1.9 * INSN_COST);
12222 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12223
12224 ins_encode %{
12225 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12226 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12227 %}
12228 ins_pipe(ialu_reg_reg_shift);
12229 %}
12230
12231 // This pattern is automatically generated from aarch64_ad.m4
12232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12233 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12234 %{
12235 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12236 ins_cost(1.9 * INSN_COST);
12237 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12238
12239 ins_encode %{
12240 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12241 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12242 %}
12243 ins_pipe(ialu_reg_reg_shift);
12244 %}
12245
12246 // This pattern is automatically generated from aarch64_ad.m4
12247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12248 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12249 %{
12250 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12251 ins_cost(1.9 * INSN_COST);
12252 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12253
12254 ins_encode %{
12255 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12256 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12257 %}
12258 ins_pipe(ialu_reg_reg_shift);
12259 %}
12260
12261 // This pattern is automatically generated from aarch64_ad.m4
12262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12263 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12264 %{
12265 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12266 ins_cost(1.9 * INSN_COST);
12267 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12268
12269 ins_encode %{
12270 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12271 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12272 %}
12273 ins_pipe(ialu_reg_reg_shift);
12274 %}
12275
12276 // This pattern is automatically generated from aarch64_ad.m4
12277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12278 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12279 %{
12280 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12281 ins_cost(1.9 * INSN_COST);
12282 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12283
12284 ins_encode %{
12285 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12286 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12287 %}
12288 ins_pipe(ialu_reg_reg_shift);
12289 %}
12290
12291 // This pattern is automatically generated from aarch64_ad.m4
12292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12293 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12294 %{
12295 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12296 ins_cost(1.9 * INSN_COST);
12297 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12298
12299 ins_encode %{
12300 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12301 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12302 %}
12303 ins_pipe(ialu_reg_reg_shift);
12304 %}
12305
12306 // This pattern is automatically generated from aarch64_ad.m4
12307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12308 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12309 %{
12310 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12311 ins_cost(1.9 * INSN_COST);
12312 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12313
12314 ins_encode %{
12315 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12316 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12317 %}
12318 ins_pipe(ialu_reg_reg_shift);
12319 %}
12320
12321 // This pattern is automatically generated from aarch64_ad.m4
12322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12323 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12324 %{
12325 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12326 ins_cost(1.9 * INSN_COST);
12327 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12328
12329 ins_encode %{
12330 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12331 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12332 %}
12333 ins_pipe(ialu_reg_reg_shift);
12334 %}
12335
12336 // This pattern is automatically generated from aarch64_ad.m4
12337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12338 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12339 %{
12340 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12341 ins_cost(1.9 * INSN_COST);
12342 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12343
12344 ins_encode %{
12345 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12346 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12347 %}
12348 ins_pipe(ialu_reg_reg_shift);
12349 %}
12350
12351 // This pattern is automatically generated from aarch64_ad.m4
12352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12353 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12354 %{
12355 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12356 ins_cost(1.9 * INSN_COST);
12357 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12358
12359 ins_encode %{
12360 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12361 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12362 %}
12363 ins_pipe(ialu_reg_reg_shift);
12364 %}
12365
12366 // This pattern is automatically generated from aarch64_ad.m4
12367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12368 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12369 %{
12370 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12371 ins_cost(1.9 * INSN_COST);
12372 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12373
12374 ins_encode %{
12375 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12376 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12377 %}
12378 ins_pipe(ialu_reg_reg_shift);
12379 %}
12380
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12384 %{
12385 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12386 ins_cost(1.9 * INSN_COST);
12387 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12388
12389 ins_encode %{
12390 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12391 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12392 %}
12393 ins_pipe(ialu_reg_reg_shift);
12394 %}
12395
12396 // This pattern is automatically generated from aarch64_ad.m4
12397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12398 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12399 %{
12400 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12401 ins_cost(1.9 * INSN_COST);
12402 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12403
12404 ins_encode %{
12405 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12406 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12407 %}
12408 ins_pipe(ialu_reg_reg_shift);
12409 %}
12410
12411 // This pattern is automatically generated from aarch64_ad.m4
12412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12413 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12414 %{
12415 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12416 ins_cost(1.9 * INSN_COST);
12417 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12418
12419 ins_encode %{
12420 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12421 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12422 %}
12423 ins_pipe(ialu_reg_reg_shift);
12424 %}
12425
12426 // This pattern is automatically generated from aarch64_ad.m4
12427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12428 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12429 %{
12430 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12431 ins_cost(1.9 * INSN_COST);
12432 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12433
12434 ins_encode %{
12435 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12436 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12437 %}
12438 ins_pipe(ialu_reg_reg_shift);
12439 %}
12440
12441 // This pattern is automatically generated from aarch64_ad.m4
12442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12443 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12444 %{
12445 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12446 ins_cost(1.9 * INSN_COST);
12447 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12448
12449 ins_encode %{
12450 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12451 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12452 %}
12453 ins_pipe(ialu_reg_reg_shift);
12454 %}
12455
12456 // This pattern is automatically generated from aarch64_ad.m4
12457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12458 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12459 %{
12460 effect(DEF dst, USE src1, USE src2, USE cr);
12461 ins_cost(INSN_COST * 2);
12462 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12463
12464 ins_encode %{
12465 __ cselw($dst$$Register,
12466 $src1$$Register,
12467 $src2$$Register,
12468 Assembler::LT);
12469 %}
12470 ins_pipe(icond_reg_reg);
12471 %}
12472
12473 // This pattern is automatically generated from aarch64_ad.m4
12474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12475 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12476 %{
12477 effect(DEF dst, USE src1, USE src2, USE cr);
12478 ins_cost(INSN_COST * 2);
12479 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12480
12481 ins_encode %{
12482 __ cselw($dst$$Register,
12483 $src1$$Register,
12484 $src2$$Register,
12485 Assembler::GT);
12486 %}
12487 ins_pipe(icond_reg_reg);
12488 %}
12489
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12493 %{
12494 effect(DEF dst, USE src1, USE cr);
12495 ins_cost(INSN_COST * 2);
12496 format %{ "cselw $dst, $src1, zr lt\t" %}
12497
12498 ins_encode %{
12499 __ cselw($dst$$Register,
12500 $src1$$Register,
12501 zr,
12502 Assembler::LT);
12503 %}
12504 ins_pipe(icond_reg);
12505 %}
12506
12507 // This pattern is automatically generated from aarch64_ad.m4
12508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12509 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12510 %{
12511 effect(DEF dst, USE src1, USE cr);
12512 ins_cost(INSN_COST * 2);
12513 format %{ "cselw $dst, $src1, zr gt\t" %}
12514
12515 ins_encode %{
12516 __ cselw($dst$$Register,
12517 $src1$$Register,
12518 zr,
12519 Assembler::GT);
12520 %}
12521 ins_pipe(icond_reg);
12522 %}
12523
12524 // This pattern is automatically generated from aarch64_ad.m4
12525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12526 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12527 %{
12528 effect(DEF dst, USE src1, USE cr);
12529 ins_cost(INSN_COST * 2);
12530 format %{ "csincw $dst, $src1, zr le\t" %}
12531
12532 ins_encode %{
12533 __ csincw($dst$$Register,
12534 $src1$$Register,
12535 zr,
12536 Assembler::LE);
12537 %}
12538 ins_pipe(icond_reg);
12539 %}
12540
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12544 %{
12545 effect(DEF dst, USE src1, USE cr);
12546 ins_cost(INSN_COST * 2);
12547 format %{ "csincw $dst, $src1, zr gt\t" %}
12548
12549 ins_encode %{
12550 __ csincw($dst$$Register,
12551 $src1$$Register,
12552 zr,
12553 Assembler::GT);
12554 %}
12555 ins_pipe(icond_reg);
12556 %}
12557
12558 // This pattern is automatically generated from aarch64_ad.m4
12559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12560 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12561 %{
12562 effect(DEF dst, USE src1, USE cr);
12563 ins_cost(INSN_COST * 2);
12564 format %{ "csinvw $dst, $src1, zr lt\t" %}
12565
12566 ins_encode %{
12567 __ csinvw($dst$$Register,
12568 $src1$$Register,
12569 zr,
12570 Assembler::LT);
12571 %}
12572 ins_pipe(icond_reg);
12573 %}
12574
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12578 %{
12579 effect(DEF dst, USE src1, USE cr);
12580 ins_cost(INSN_COST * 2);
12581 format %{ "csinvw $dst, $src1, zr ge\t" %}
12582
12583 ins_encode %{
12584 __ csinvw($dst$$Register,
12585 $src1$$Register,
12586 zr,
12587 Assembler::GE);
12588 %}
12589 ins_pipe(icond_reg);
12590 %}
12591
12592 // This pattern is automatically generated from aarch64_ad.m4
12593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12594 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12595 %{
12596 match(Set dst (MinI src imm));
12597 ins_cost(INSN_COST * 3);
12598 expand %{
12599 rFlagsReg cr;
12600 compI_reg_imm0(cr, src);
12601 cmovI_reg_imm0_lt(dst, src, cr);
12602 %}
12603 %}
12604
12605 // This pattern is automatically generated from aarch64_ad.m4
12606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12607 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12608 %{
12609 match(Set dst (MinI imm src));
12610 ins_cost(INSN_COST * 3);
12611 expand %{
12612 rFlagsReg cr;
12613 compI_reg_imm0(cr, src);
12614 cmovI_reg_imm0_lt(dst, src, cr);
12615 %}
12616 %}
12617
12618 // This pattern is automatically generated from aarch64_ad.m4
12619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12620 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12621 %{
12622 match(Set dst (MinI src imm));
12623 ins_cost(INSN_COST * 3);
12624 expand %{
12625 rFlagsReg cr;
12626 compI_reg_imm0(cr, src);
12627 cmovI_reg_imm1_le(dst, src, cr);
12628 %}
12629 %}
12630
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12634 %{
12635 match(Set dst (MinI imm src));
12636 ins_cost(INSN_COST * 3);
12637 expand %{
12638 rFlagsReg cr;
12639 compI_reg_imm0(cr, src);
12640 cmovI_reg_imm1_le(dst, src, cr);
12641 %}
12642 %}
12643
12644 // This pattern is automatically generated from aarch64_ad.m4
12645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12646 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12647 %{
12648 match(Set dst (MinI src imm));
12649 ins_cost(INSN_COST * 3);
12650 expand %{
12651 rFlagsReg cr;
12652 compI_reg_imm0(cr, src);
12653 cmovI_reg_immM1_lt(dst, src, cr);
12654 %}
12655 %}
12656
12657 // This pattern is automatically generated from aarch64_ad.m4
12658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12659 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12660 %{
12661 match(Set dst (MinI imm src));
12662 ins_cost(INSN_COST * 3);
12663 expand %{
12664 rFlagsReg cr;
12665 compI_reg_imm0(cr, src);
12666 cmovI_reg_immM1_lt(dst, src, cr);
12667 %}
12668 %}
12669
12670 // This pattern is automatically generated from aarch64_ad.m4
12671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12672 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12673 %{
12674 match(Set dst (MaxI src imm));
12675 ins_cost(INSN_COST * 3);
12676 expand %{
12677 rFlagsReg cr;
12678 compI_reg_imm0(cr, src);
12679 cmovI_reg_imm0_gt(dst, src, cr);
12680 %}
12681 %}
12682
12683 // This pattern is automatically generated from aarch64_ad.m4
12684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12685 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12686 %{
12687 match(Set dst (MaxI imm src));
12688 ins_cost(INSN_COST * 3);
12689 expand %{
12690 rFlagsReg cr;
12691 compI_reg_imm0(cr, src);
12692 cmovI_reg_imm0_gt(dst, src, cr);
12693 %}
12694 %}
12695
12696 // This pattern is automatically generated from aarch64_ad.m4
12697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12698 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12699 %{
12700 match(Set dst (MaxI src imm));
12701 ins_cost(INSN_COST * 3);
12702 expand %{
12703 rFlagsReg cr;
12704 compI_reg_imm0(cr, src);
12705 cmovI_reg_imm1_gt(dst, src, cr);
12706 %}
12707 %}
12708
12709 // This pattern is automatically generated from aarch64_ad.m4
12710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12711 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12712 %{
12713 match(Set dst (MaxI imm src));
12714 ins_cost(INSN_COST * 3);
12715 expand %{
12716 rFlagsReg cr;
12717 compI_reg_imm0(cr, src);
12718 cmovI_reg_imm1_gt(dst, src, cr);
12719 %}
12720 %}
12721
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12725 %{
12726 match(Set dst (MaxI src imm));
12727 ins_cost(INSN_COST * 3);
12728 expand %{
12729 rFlagsReg cr;
12730 compI_reg_imm0(cr, src);
12731 cmovI_reg_immM1_ge(dst, src, cr);
12732 %}
12733 %}
12734
12735 // This pattern is automatically generated from aarch64_ad.m4
12736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12737 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12738 %{
12739 match(Set dst (MaxI imm src));
12740 ins_cost(INSN_COST * 3);
12741 expand %{
12742 rFlagsReg cr;
12743 compI_reg_imm0(cr, src);
12744 cmovI_reg_immM1_ge(dst, src, cr);
12745 %}
12746 %}
12747
12748 // This pattern is automatically generated from aarch64_ad.m4
12749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12750 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12751 %{
12752 match(Set dst (ReverseI src));
12753 ins_cost(INSN_COST);
12754 format %{ "rbitw $dst, $src" %}
12755 ins_encode %{
12756 __ rbitw($dst$$Register, $src$$Register);
12757 %}
12758 ins_pipe(ialu_reg);
12759 %}
12760
12761 // This pattern is automatically generated from aarch64_ad.m4
12762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12763 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12764 %{
12765 match(Set dst (ReverseL src));
12766 ins_cost(INSN_COST);
12767 format %{ "rbit $dst, $src" %}
12768 ins_encode %{
12769 __ rbit($dst$$Register, $src$$Register);
12770 %}
12771 ins_pipe(ialu_reg);
12772 %}
12773
12774
12775 // END This section of the file is automatically generated. Do not edit --------------
12776
12777
12778 // ============================================================================
12779 // Floating Point Arithmetic Instructions
12780
12781 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12782 match(Set dst (AddHF src1 src2));
12783 format %{ "faddh $dst, $src1, $src2" %}
12784 ins_encode %{
12785 __ faddh($dst$$FloatRegister,
12786 $src1$$FloatRegister,
12787 $src2$$FloatRegister);
12788 %}
12789 ins_pipe(fp_dop_reg_reg_s);
12790 %}
12791
12792 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12793 match(Set dst (AddF src1 src2));
12794
12795 ins_cost(INSN_COST * 5);
12796 format %{ "fadds $dst, $src1, $src2" %}
12797
12798 ins_encode %{
12799 __ fadds(as_FloatRegister($dst$$reg),
12800 as_FloatRegister($src1$$reg),
12801 as_FloatRegister($src2$$reg));
12802 %}
12803
12804 ins_pipe(fp_dop_reg_reg_s);
12805 %}
12806
12807 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12808 match(Set dst (AddD src1 src2));
12809
12810 ins_cost(INSN_COST * 5);
12811 format %{ "faddd $dst, $src1, $src2" %}
12812
12813 ins_encode %{
12814 __ faddd(as_FloatRegister($dst$$reg),
12815 as_FloatRegister($src1$$reg),
12816 as_FloatRegister($src2$$reg));
12817 %}
12818
12819 ins_pipe(fp_dop_reg_reg_d);
12820 %}
12821
12822 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12823 match(Set dst (SubHF src1 src2));
12824 format %{ "fsubh $dst, $src1, $src2" %}
12825 ins_encode %{
12826 __ fsubh($dst$$FloatRegister,
12827 $src1$$FloatRegister,
12828 $src2$$FloatRegister);
12829 %}
12830 ins_pipe(fp_dop_reg_reg_s);
12831 %}
12832
12833 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12834 match(Set dst (SubF src1 src2));
12835
12836 ins_cost(INSN_COST * 5);
12837 format %{ "fsubs $dst, $src1, $src2" %}
12838
12839 ins_encode %{
12840 __ fsubs(as_FloatRegister($dst$$reg),
12841 as_FloatRegister($src1$$reg),
12842 as_FloatRegister($src2$$reg));
12843 %}
12844
12845 ins_pipe(fp_dop_reg_reg_s);
12846 %}
12847
12848 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12849 match(Set dst (SubD src1 src2));
12850
12851 ins_cost(INSN_COST * 5);
12852 format %{ "fsubd $dst, $src1, $src2" %}
12853
12854 ins_encode %{
12855 __ fsubd(as_FloatRegister($dst$$reg),
12856 as_FloatRegister($src1$$reg),
12857 as_FloatRegister($src2$$reg));
12858 %}
12859
12860 ins_pipe(fp_dop_reg_reg_d);
12861 %}
12862
12863 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12864 match(Set dst (MulHF src1 src2));
12865 format %{ "fmulh $dst, $src1, $src2" %}
12866 ins_encode %{
12867 __ fmulh($dst$$FloatRegister,
12868 $src1$$FloatRegister,
12869 $src2$$FloatRegister);
12870 %}
12871 ins_pipe(fp_dop_reg_reg_s);
12872 %}
12873
12874 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12875 match(Set dst (MulF src1 src2));
12876
12877 ins_cost(INSN_COST * 6);
12878 format %{ "fmuls $dst, $src1, $src2" %}
12879
12880 ins_encode %{
12881 __ fmuls(as_FloatRegister($dst$$reg),
12882 as_FloatRegister($src1$$reg),
12883 as_FloatRegister($src2$$reg));
12884 %}
12885
12886 ins_pipe(fp_dop_reg_reg_s);
12887 %}
12888
12889 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12890 match(Set dst (MulD src1 src2));
12891
12892 ins_cost(INSN_COST * 6);
12893 format %{ "fmuld $dst, $src1, $src2" %}
12894
12895 ins_encode %{
12896 __ fmuld(as_FloatRegister($dst$$reg),
12897 as_FloatRegister($src1$$reg),
12898 as_FloatRegister($src2$$reg));
12899 %}
12900
12901 ins_pipe(fp_dop_reg_reg_d);
12902 %}
12903
12904 // src1 * src2 + src3 (half-precision float)
12905 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12906 match(Set dst (FmaHF src3 (Binary src1 src2)));
12907 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12908 ins_encode %{
12909 assert(UseFMA, "Needs FMA instructions support.");
12910 __ fmaddh($dst$$FloatRegister,
12911 $src1$$FloatRegister,
12912 $src2$$FloatRegister,
12913 $src3$$FloatRegister);
12914 %}
12915 ins_pipe(pipe_class_default);
12916 %}
12917
12918 // src1 * src2 + src3
12919 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12920 match(Set dst (FmaF src3 (Binary src1 src2)));
12921
12922 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12923
12924 ins_encode %{
12925 assert(UseFMA, "Needs FMA instructions support.");
12926 __ fmadds(as_FloatRegister($dst$$reg),
12927 as_FloatRegister($src1$$reg),
12928 as_FloatRegister($src2$$reg),
12929 as_FloatRegister($src3$$reg));
12930 %}
12931
12932 ins_pipe(pipe_class_default);
12933 %}
12934
12935 // src1 * src2 + src3
12936 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12937 match(Set dst (FmaD src3 (Binary src1 src2)));
12938
12939 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12940
12941 ins_encode %{
12942 assert(UseFMA, "Needs FMA instructions support.");
12943 __ fmaddd(as_FloatRegister($dst$$reg),
12944 as_FloatRegister($src1$$reg),
12945 as_FloatRegister($src2$$reg),
12946 as_FloatRegister($src3$$reg));
12947 %}
12948
12949 ins_pipe(pipe_class_default);
12950 %}
12951
12952 // src1 * (-src2) + src3
12953 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12954 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12955 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12956
12957 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12958
12959 ins_encode %{
12960 assert(UseFMA, "Needs FMA instructions support.");
12961 __ fmsubs(as_FloatRegister($dst$$reg),
12962 as_FloatRegister($src1$$reg),
12963 as_FloatRegister($src2$$reg),
12964 as_FloatRegister($src3$$reg));
12965 %}
12966
12967 ins_pipe(pipe_class_default);
12968 %}
12969
12970 // src1 * (-src2) + src3
12971 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12972 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12973 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12974
12975 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12976
12977 ins_encode %{
12978 assert(UseFMA, "Needs FMA instructions support.");
12979 __ fmsubd(as_FloatRegister($dst$$reg),
12980 as_FloatRegister($src1$$reg),
12981 as_FloatRegister($src2$$reg),
12982 as_FloatRegister($src3$$reg));
12983 %}
12984
12985 ins_pipe(pipe_class_default);
12986 %}
12987
12988 // src1 * (-src2) - src3
12989 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12990 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12991 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12992
12993 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
12994
12995 ins_encode %{
12996 assert(UseFMA, "Needs FMA instructions support.");
12997 __ fnmadds(as_FloatRegister($dst$$reg),
12998 as_FloatRegister($src1$$reg),
12999 as_FloatRegister($src2$$reg),
13000 as_FloatRegister($src3$$reg));
13001 %}
13002
13003 ins_pipe(pipe_class_default);
13004 %}
13005
13006 // src1 * (-src2) - src3
13007 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13008 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13009 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13010
13011 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13012
13013 ins_encode %{
13014 assert(UseFMA, "Needs FMA instructions support.");
13015 __ fnmaddd(as_FloatRegister($dst$$reg),
13016 as_FloatRegister($src1$$reg),
13017 as_FloatRegister($src2$$reg),
13018 as_FloatRegister($src3$$reg));
13019 %}
13020
13021 ins_pipe(pipe_class_default);
13022 %}
13023
13024 // src1 * src2 - src3
13025 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13026 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13027
13028 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13029
13030 ins_encode %{
13031 assert(UseFMA, "Needs FMA instructions support.");
13032 __ fnmsubs(as_FloatRegister($dst$$reg),
13033 as_FloatRegister($src1$$reg),
13034 as_FloatRegister($src2$$reg),
13035 as_FloatRegister($src3$$reg));
13036 %}
13037
13038 ins_pipe(pipe_class_default);
13039 %}
13040
13041 // src1 * src2 - src3
13042 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13043 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13044
13045 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13046
13047 ins_encode %{
13048 assert(UseFMA, "Needs FMA instructions support.");
13049 // n.b. insn name should be fnmsubd
13050 __ fnmsub(as_FloatRegister($dst$$reg),
13051 as_FloatRegister($src1$$reg),
13052 as_FloatRegister($src2$$reg),
13053 as_FloatRegister($src3$$reg));
13054 %}
13055
13056 ins_pipe(pipe_class_default);
13057 %}
13058
13059 // Math.max(HH)H (half-precision float)
13060 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13061 match(Set dst (MaxHF src1 src2));
13062 format %{ "fmaxh $dst, $src1, $src2" %}
13063 ins_encode %{
13064 __ fmaxh($dst$$FloatRegister,
13065 $src1$$FloatRegister,
13066 $src2$$FloatRegister);
13067 %}
13068 ins_pipe(fp_dop_reg_reg_s);
13069 %}
13070
13071 // Math.min(HH)H (half-precision float)
13072 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13073 match(Set dst (MinHF src1 src2));
13074 format %{ "fminh $dst, $src1, $src2" %}
13075 ins_encode %{
13076 __ fminh($dst$$FloatRegister,
13077 $src1$$FloatRegister,
13078 $src2$$FloatRegister);
13079 %}
13080 ins_pipe(fp_dop_reg_reg_s);
13081 %}
13082
13083 // Math.max(FF)F
13084 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13085 match(Set dst (MaxF src1 src2));
13086
13087 format %{ "fmaxs $dst, $src1, $src2" %}
13088 ins_encode %{
13089 __ fmaxs(as_FloatRegister($dst$$reg),
13090 as_FloatRegister($src1$$reg),
13091 as_FloatRegister($src2$$reg));
13092 %}
13093
13094 ins_pipe(fp_dop_reg_reg_s);
13095 %}
13096
13097 // Math.min(FF)F
13098 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13099 match(Set dst (MinF src1 src2));
13100
13101 format %{ "fmins $dst, $src1, $src2" %}
13102 ins_encode %{
13103 __ fmins(as_FloatRegister($dst$$reg),
13104 as_FloatRegister($src1$$reg),
13105 as_FloatRegister($src2$$reg));
13106 %}
13107
13108 ins_pipe(fp_dop_reg_reg_s);
13109 %}
13110
13111 // Math.max(DD)D
13112 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13113 match(Set dst (MaxD src1 src2));
13114
13115 format %{ "fmaxd $dst, $src1, $src2" %}
13116 ins_encode %{
13117 __ fmaxd(as_FloatRegister($dst$$reg),
13118 as_FloatRegister($src1$$reg),
13119 as_FloatRegister($src2$$reg));
13120 %}
13121
13122 ins_pipe(fp_dop_reg_reg_d);
13123 %}
13124
13125 // Math.min(DD)D
13126 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13127 match(Set dst (MinD src1 src2));
13128
13129 format %{ "fmind $dst, $src1, $src2" %}
13130 ins_encode %{
13131 __ fmind(as_FloatRegister($dst$$reg),
13132 as_FloatRegister($src1$$reg),
13133 as_FloatRegister($src2$$reg));
13134 %}
13135
13136 ins_pipe(fp_dop_reg_reg_d);
13137 %}
13138
13139 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13140 match(Set dst (DivHF src1 src2));
13141 format %{ "fdivh $dst, $src1, $src2" %}
13142 ins_encode %{
13143 __ fdivh($dst$$FloatRegister,
13144 $src1$$FloatRegister,
13145 $src2$$FloatRegister);
13146 %}
13147 ins_pipe(fp_div_s);
13148 %}
13149
13150 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13151 match(Set dst (DivF src1 src2));
13152
13153 ins_cost(INSN_COST * 18);
13154 format %{ "fdivs $dst, $src1, $src2" %}
13155
13156 ins_encode %{
13157 __ fdivs(as_FloatRegister($dst$$reg),
13158 as_FloatRegister($src1$$reg),
13159 as_FloatRegister($src2$$reg));
13160 %}
13161
13162 ins_pipe(fp_div_s);
13163 %}
13164
13165 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13166 match(Set dst (DivD src1 src2));
13167
13168 ins_cost(INSN_COST * 32);
13169 format %{ "fdivd $dst, $src1, $src2" %}
13170
13171 ins_encode %{
13172 __ fdivd(as_FloatRegister($dst$$reg),
13173 as_FloatRegister($src1$$reg),
13174 as_FloatRegister($src2$$reg));
13175 %}
13176
13177 ins_pipe(fp_div_d);
13178 %}
13179
13180 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13181 match(Set dst (NegF src));
13182
13183 ins_cost(INSN_COST * 3);
13184 format %{ "fneg $dst, $src" %}
13185
13186 ins_encode %{
13187 __ fnegs(as_FloatRegister($dst$$reg),
13188 as_FloatRegister($src$$reg));
13189 %}
13190
13191 ins_pipe(fp_uop_s);
13192 %}
13193
13194 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13195 match(Set dst (NegD src));
13196
13197 ins_cost(INSN_COST * 3);
13198 format %{ "fnegd $dst, $src" %}
13199
13200 ins_encode %{
13201 __ fnegd(as_FloatRegister($dst$$reg),
13202 as_FloatRegister($src$$reg));
13203 %}
13204
13205 ins_pipe(fp_uop_d);
13206 %}
13207
13208 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13209 %{
13210 match(Set dst (AbsI src));
13211
13212 effect(KILL cr);
13213 ins_cost(INSN_COST * 2);
13214 format %{ "cmpw $src, zr\n\t"
13215 "cnegw $dst, $src, Assembler::LT\t# int abs"
13216 %}
13217
13218 ins_encode %{
13219 __ cmpw(as_Register($src$$reg), zr);
13220 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13221 %}
13222 ins_pipe(pipe_class_default);
13223 %}
13224
13225 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13226 %{
13227 match(Set dst (AbsL src));
13228
13229 effect(KILL cr);
13230 ins_cost(INSN_COST * 2);
13231 format %{ "cmp $src, zr\n\t"
13232 "cneg $dst, $src, Assembler::LT\t# long abs"
13233 %}
13234
13235 ins_encode %{
13236 __ cmp(as_Register($src$$reg), zr);
13237 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13238 %}
13239 ins_pipe(pipe_class_default);
13240 %}
13241
13242 instruct absF_reg(vRegF dst, vRegF src) %{
13243 match(Set dst (AbsF src));
13244
13245 ins_cost(INSN_COST * 3);
13246 format %{ "fabss $dst, $src" %}
13247 ins_encode %{
13248 __ fabss(as_FloatRegister($dst$$reg),
13249 as_FloatRegister($src$$reg));
13250 %}
13251
13252 ins_pipe(fp_uop_s);
13253 %}
13254
13255 instruct absD_reg(vRegD dst, vRegD src) %{
13256 match(Set dst (AbsD src));
13257
13258 ins_cost(INSN_COST * 3);
13259 format %{ "fabsd $dst, $src" %}
13260 ins_encode %{
13261 __ fabsd(as_FloatRegister($dst$$reg),
13262 as_FloatRegister($src$$reg));
13263 %}
13264
13265 ins_pipe(fp_uop_d);
13266 %}
13267
13268 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13269 match(Set dst (AbsF (SubF src1 src2)));
13270
13271 ins_cost(INSN_COST * 3);
13272 format %{ "fabds $dst, $src1, $src2" %}
13273 ins_encode %{
13274 __ fabds(as_FloatRegister($dst$$reg),
13275 as_FloatRegister($src1$$reg),
13276 as_FloatRegister($src2$$reg));
13277 %}
13278
13279 ins_pipe(fp_uop_s);
13280 %}
13281
13282 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13283 match(Set dst (AbsD (SubD src1 src2)));
13284
13285 ins_cost(INSN_COST * 3);
13286 format %{ "fabdd $dst, $src1, $src2" %}
13287 ins_encode %{
13288 __ fabdd(as_FloatRegister($dst$$reg),
13289 as_FloatRegister($src1$$reg),
13290 as_FloatRegister($src2$$reg));
13291 %}
13292
13293 ins_pipe(fp_uop_d);
13294 %}
13295
13296 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13297 match(Set dst (SqrtD src));
13298
13299 ins_cost(INSN_COST * 50);
13300 format %{ "fsqrtd $dst, $src" %}
13301 ins_encode %{
13302 __ fsqrtd(as_FloatRegister($dst$$reg),
13303 as_FloatRegister($src$$reg));
13304 %}
13305
13306 ins_pipe(fp_div_s);
13307 %}
13308
13309 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13310 match(Set dst (SqrtF src));
13311
13312 ins_cost(INSN_COST * 50);
13313 format %{ "fsqrts $dst, $src" %}
13314 ins_encode %{
13315 __ fsqrts(as_FloatRegister($dst$$reg),
13316 as_FloatRegister($src$$reg));
13317 %}
13318
13319 ins_pipe(fp_div_d);
13320 %}
13321
13322 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13323 match(Set dst (SqrtHF src));
13324 format %{ "fsqrth $dst, $src" %}
13325 ins_encode %{
13326 __ fsqrth($dst$$FloatRegister,
13327 $src$$FloatRegister);
13328 %}
13329 ins_pipe(fp_div_s);
13330 %}
13331
13332 // Math.rint, floor, ceil
13333 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13334 match(Set dst (RoundDoubleMode src rmode));
13335 format %{ "frint $dst, $src, $rmode" %}
13336 ins_encode %{
13337 switch ($rmode$$constant) {
13338 case RoundDoubleModeNode::rmode_rint:
13339 __ frintnd(as_FloatRegister($dst$$reg),
13340 as_FloatRegister($src$$reg));
13341 break;
13342 case RoundDoubleModeNode::rmode_floor:
13343 __ frintmd(as_FloatRegister($dst$$reg),
13344 as_FloatRegister($src$$reg));
13345 break;
13346 case RoundDoubleModeNode::rmode_ceil:
13347 __ frintpd(as_FloatRegister($dst$$reg),
13348 as_FloatRegister($src$$reg));
13349 break;
13350 }
13351 %}
13352 ins_pipe(fp_uop_d);
13353 %}
13354
13355 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13356 match(Set dst (CopySignD src1 (Binary src2 zero)));
13357 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13358 format %{ "CopySignD $dst $src1 $src2" %}
13359 ins_encode %{
13360 FloatRegister dst = as_FloatRegister($dst$$reg),
13361 src1 = as_FloatRegister($src1$$reg),
13362 src2 = as_FloatRegister($src2$$reg),
13363 zero = as_FloatRegister($zero$$reg);
13364 __ fnegd(dst, zero);
13365 __ bsl(dst, __ T8B, src2, src1);
13366 %}
13367 ins_pipe(fp_uop_d);
13368 %}
13369
13370 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13371 match(Set dst (CopySignF src1 src2));
13372 effect(TEMP_DEF dst, USE src1, USE src2);
13373 format %{ "CopySignF $dst $src1 $src2" %}
13374 ins_encode %{
13375 FloatRegister dst = as_FloatRegister($dst$$reg),
13376 src1 = as_FloatRegister($src1$$reg),
13377 src2 = as_FloatRegister($src2$$reg);
13378 __ movi(dst, __ T2S, 0x80, 24);
13379 __ bsl(dst, __ T8B, src2, src1);
13380 %}
13381 ins_pipe(fp_uop_d);
13382 %}
13383
13384 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13385 match(Set dst (SignumD src (Binary zero one)));
13386 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13387 format %{ "signumD $dst, $src" %}
13388 ins_encode %{
13389 FloatRegister src = as_FloatRegister($src$$reg),
13390 dst = as_FloatRegister($dst$$reg),
13391 zero = as_FloatRegister($zero$$reg),
13392 one = as_FloatRegister($one$$reg);
13393 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13394 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13395 // Bit selection instruction gets bit from "one" for each enabled bit in
13396 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13397 // NaN the whole "src" will be copied because "dst" is zero. For all other
13398 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13399 // from "src", and all other bits are copied from 1.0.
13400 __ bsl(dst, __ T8B, one, src);
13401 %}
13402 ins_pipe(fp_uop_d);
13403 %}
13404
13405 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13406 match(Set dst (SignumF src (Binary zero one)));
13407 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13408 format %{ "signumF $dst, $src" %}
13409 ins_encode %{
13410 FloatRegister src = as_FloatRegister($src$$reg),
13411 dst = as_FloatRegister($dst$$reg),
13412 zero = as_FloatRegister($zero$$reg),
13413 one = as_FloatRegister($one$$reg);
13414 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13415 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13416 // Bit selection instruction gets bit from "one" for each enabled bit in
13417 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13418 // NaN the whole "src" will be copied because "dst" is zero. For all other
13419 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13420 // from "src", and all other bits are copied from 1.0.
13421 __ bsl(dst, __ T8B, one, src);
13422 %}
13423 ins_pipe(fp_uop_d);
13424 %}
13425
13426 instruct onspinwait() %{
13427 match(OnSpinWait);
13428 ins_cost(INSN_COST);
13429
13430 format %{ "onspinwait" %}
13431
13432 ins_encode %{
13433 __ spin_wait();
13434 %}
13435 ins_pipe(pipe_class_empty);
13436 %}
13437
13438 // ============================================================================
13439 // Logical Instructions
13440
13441 // Integer Logical Instructions
13442
13443 // And Instructions
13444
13445
13446 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13447 match(Set dst (AndI src1 src2));
13448
13449 format %{ "andw $dst, $src1, $src2\t# int" %}
13450
13451 ins_cost(INSN_COST);
13452 ins_encode %{
13453 __ andw(as_Register($dst$$reg),
13454 as_Register($src1$$reg),
13455 as_Register($src2$$reg));
13456 %}
13457
13458 ins_pipe(ialu_reg_reg);
13459 %}
13460
13461 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13462 match(Set dst (AndI src1 src2));
13463
13464 format %{ "andsw $dst, $src1, $src2\t# int" %}
13465
13466 ins_cost(INSN_COST);
13467 ins_encode %{
13468 __ andw(as_Register($dst$$reg),
13469 as_Register($src1$$reg),
13470 (uint64_t)($src2$$constant));
13471 %}
13472
13473 ins_pipe(ialu_reg_imm);
13474 %}
13475
13476 // Or Instructions
13477
13478 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13479 match(Set dst (OrI src1 src2));
13480
13481 format %{ "orrw $dst, $src1, $src2\t# int" %}
13482
13483 ins_cost(INSN_COST);
13484 ins_encode %{
13485 __ orrw(as_Register($dst$$reg),
13486 as_Register($src1$$reg),
13487 as_Register($src2$$reg));
13488 %}
13489
13490 ins_pipe(ialu_reg_reg);
13491 %}
13492
13493 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13494 match(Set dst (OrI src1 src2));
13495
13496 format %{ "orrw $dst, $src1, $src2\t# int" %}
13497
13498 ins_cost(INSN_COST);
13499 ins_encode %{
13500 __ orrw(as_Register($dst$$reg),
13501 as_Register($src1$$reg),
13502 (uint64_t)($src2$$constant));
13503 %}
13504
13505 ins_pipe(ialu_reg_imm);
13506 %}
13507
13508 // Xor Instructions
13509
13510 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13511 match(Set dst (XorI src1 src2));
13512
13513 format %{ "eorw $dst, $src1, $src2\t# int" %}
13514
13515 ins_cost(INSN_COST);
13516 ins_encode %{
13517 __ eorw(as_Register($dst$$reg),
13518 as_Register($src1$$reg),
13519 as_Register($src2$$reg));
13520 %}
13521
13522 ins_pipe(ialu_reg_reg);
13523 %}
13524
13525 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13526 match(Set dst (XorI src1 src2));
13527
13528 format %{ "eorw $dst, $src1, $src2\t# int" %}
13529
13530 ins_cost(INSN_COST);
13531 ins_encode %{
13532 __ eorw(as_Register($dst$$reg),
13533 as_Register($src1$$reg),
13534 (uint64_t)($src2$$constant));
13535 %}
13536
13537 ins_pipe(ialu_reg_imm);
13538 %}
13539
13540 // Long Logical Instructions
13541 // TODO
13542
13543 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13544 match(Set dst (AndL src1 src2));
13545
13546 format %{ "and $dst, $src1, $src2\t# int" %}
13547
13548 ins_cost(INSN_COST);
13549 ins_encode %{
13550 __ andr(as_Register($dst$$reg),
13551 as_Register($src1$$reg),
13552 as_Register($src2$$reg));
13553 %}
13554
13555 ins_pipe(ialu_reg_reg);
13556 %}
13557
13558 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13559 match(Set dst (AndL src1 src2));
13560
13561 format %{ "and $dst, $src1, $src2\t# int" %}
13562
13563 ins_cost(INSN_COST);
13564 ins_encode %{
13565 __ andr(as_Register($dst$$reg),
13566 as_Register($src1$$reg),
13567 (uint64_t)($src2$$constant));
13568 %}
13569
13570 ins_pipe(ialu_reg_imm);
13571 %}
13572
13573 // Or Instructions
13574
13575 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13576 match(Set dst (OrL src1 src2));
13577
13578 format %{ "orr $dst, $src1, $src2\t# int" %}
13579
13580 ins_cost(INSN_COST);
13581 ins_encode %{
13582 __ orr(as_Register($dst$$reg),
13583 as_Register($src1$$reg),
13584 as_Register($src2$$reg));
13585 %}
13586
13587 ins_pipe(ialu_reg_reg);
13588 %}
13589
13590 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13591 match(Set dst (OrL src1 src2));
13592
13593 format %{ "orr $dst, $src1, $src2\t# int" %}
13594
13595 ins_cost(INSN_COST);
13596 ins_encode %{
13597 __ orr(as_Register($dst$$reg),
13598 as_Register($src1$$reg),
13599 (uint64_t)($src2$$constant));
13600 %}
13601
13602 ins_pipe(ialu_reg_imm);
13603 %}
13604
13605 // Xor Instructions
13606
13607 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13608 match(Set dst (XorL src1 src2));
13609
13610 format %{ "eor $dst, $src1, $src2\t# int" %}
13611
13612 ins_cost(INSN_COST);
13613 ins_encode %{
13614 __ eor(as_Register($dst$$reg),
13615 as_Register($src1$$reg),
13616 as_Register($src2$$reg));
13617 %}
13618
13619 ins_pipe(ialu_reg_reg);
13620 %}
13621
13622 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13623 match(Set dst (XorL src1 src2));
13624
13625 ins_cost(INSN_COST);
13626 format %{ "eor $dst, $src1, $src2\t# int" %}
13627
13628 ins_encode %{
13629 __ eor(as_Register($dst$$reg),
13630 as_Register($src1$$reg),
13631 (uint64_t)($src2$$constant));
13632 %}
13633
13634 ins_pipe(ialu_reg_imm);
13635 %}
13636
13637 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13638 %{
13639 match(Set dst (ConvI2L src));
13640
13641 ins_cost(INSN_COST);
13642 format %{ "sxtw $dst, $src\t# i2l" %}
13643 ins_encode %{
13644 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13645 %}
13646 ins_pipe(ialu_reg_shift);
13647 %}
13648
13649 // this pattern occurs in bigmath arithmetic
13650 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13651 %{
13652 match(Set dst (AndL (ConvI2L src) mask));
13653
13654 ins_cost(INSN_COST);
13655 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13656 ins_encode %{
13657 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13658 %}
13659
13660 ins_pipe(ialu_reg_shift);
13661 %}
13662
13663 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13664 match(Set dst (ConvL2I src));
13665
13666 ins_cost(INSN_COST);
13667 format %{ "movw $dst, $src \t// l2i" %}
13668
13669 ins_encode %{
13670 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13671 %}
13672
13673 ins_pipe(ialu_reg);
13674 %}
13675
13676 instruct convD2F_reg(vRegF dst, vRegD src) %{
13677 match(Set dst (ConvD2F src));
13678
13679 ins_cost(INSN_COST * 5);
13680 format %{ "fcvtd $dst, $src \t// d2f" %}
13681
13682 ins_encode %{
13683 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13684 %}
13685
13686 ins_pipe(fp_d2f);
13687 %}
13688
13689 instruct convF2D_reg(vRegD dst, vRegF src) %{
13690 match(Set dst (ConvF2D src));
13691
13692 ins_cost(INSN_COST * 5);
13693 format %{ "fcvts $dst, $src \t// f2d" %}
13694
13695 ins_encode %{
13696 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13697 %}
13698
13699 ins_pipe(fp_f2d);
13700 %}
13701
13702 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13703 match(Set dst (ConvF2I src));
13704
13705 ins_cost(INSN_COST * 5);
13706 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13707
13708 ins_encode %{
13709 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13710 %}
13711
13712 ins_pipe(fp_f2i);
13713 %}
13714
13715 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13716 match(Set dst (ConvF2L src));
13717
13718 ins_cost(INSN_COST * 5);
13719 format %{ "fcvtzs $dst, $src \t// f2l" %}
13720
13721 ins_encode %{
13722 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13723 %}
13724
13725 ins_pipe(fp_f2l);
13726 %}
13727
13728 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13729 match(Set dst (ConvF2HF src));
13730 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13731 "smov $dst, $tmp\t# move result from $tmp to $dst"
13732 %}
13733 effect(TEMP tmp);
13734 ins_encode %{
13735 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13736 %}
13737 ins_pipe(pipe_slow);
13738 %}
13739
13740 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13741 match(Set dst (ConvHF2F src));
13742 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13743 "fcvt $dst, $tmp\t# convert half to single precision"
13744 %}
13745 effect(TEMP tmp);
13746 ins_encode %{
13747 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13748 %}
13749 ins_pipe(pipe_slow);
13750 %}
13751
13752 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13753 match(Set dst (ConvI2F src));
13754
13755 ins_cost(INSN_COST * 5);
13756 format %{ "scvtfws $dst, $src \t// i2f" %}
13757
13758 ins_encode %{
13759 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13760 %}
13761
13762 ins_pipe(fp_i2f);
13763 %}
13764
13765 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13766 match(Set dst (ConvL2F src));
13767
13768 ins_cost(INSN_COST * 5);
13769 format %{ "scvtfs $dst, $src \t// l2f" %}
13770
13771 ins_encode %{
13772 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13773 %}
13774
13775 ins_pipe(fp_l2f);
13776 %}
13777
13778 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13779 match(Set dst (ConvD2I src));
13780
13781 ins_cost(INSN_COST * 5);
13782 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13783
13784 ins_encode %{
13785 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13786 %}
13787
13788 ins_pipe(fp_d2i);
13789 %}
13790
13791 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13792 match(Set dst (ConvD2L src));
13793
13794 ins_cost(INSN_COST * 5);
13795 format %{ "fcvtzd $dst, $src \t// d2l" %}
13796
13797 ins_encode %{
13798 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13799 %}
13800
13801 ins_pipe(fp_d2l);
13802 %}
13803
13804 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13805 match(Set dst (ConvI2D src));
13806
13807 ins_cost(INSN_COST * 5);
13808 format %{ "scvtfwd $dst, $src \t// i2d" %}
13809
13810 ins_encode %{
13811 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13812 %}
13813
13814 ins_pipe(fp_i2d);
13815 %}
13816
13817 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13818 match(Set dst (ConvL2D src));
13819
13820 ins_cost(INSN_COST * 5);
13821 format %{ "scvtfd $dst, $src \t// l2d" %}
13822
13823 ins_encode %{
13824 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13825 %}
13826
13827 ins_pipe(fp_l2d);
13828 %}
13829
13830 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13831 %{
13832 match(Set dst (RoundD src));
13833 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13834 format %{ "java_round_double $dst,$src"%}
13835 ins_encode %{
13836 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13837 as_FloatRegister($ftmp$$reg));
13838 %}
13839 ins_pipe(pipe_slow);
13840 %}
13841
13842 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13843 %{
13844 match(Set dst (RoundF src));
13845 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13846 format %{ "java_round_float $dst,$src"%}
13847 ins_encode %{
13848 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13849 as_FloatRegister($ftmp$$reg));
13850 %}
13851 ins_pipe(pipe_slow);
13852 %}
13853
13854 // stack <-> reg and reg <-> reg shuffles with no conversion
13855
13856 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13857
13858 match(Set dst (MoveF2I src));
13859
13860 effect(DEF dst, USE src);
13861
13862 ins_cost(4 * INSN_COST);
13863
13864 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13865
13866 ins_encode %{
13867 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13868 %}
13869
13870 ins_pipe(iload_reg_reg);
13871
13872 %}
13873
13874 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13875
13876 match(Set dst (MoveI2F src));
13877
13878 effect(DEF dst, USE src);
13879
13880 ins_cost(4 * INSN_COST);
13881
13882 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13883
13884 ins_encode %{
13885 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13886 %}
13887
13888 ins_pipe(pipe_class_memory);
13889
13890 %}
13891
13892 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13893
13894 match(Set dst (MoveD2L src));
13895
13896 effect(DEF dst, USE src);
13897
13898 ins_cost(4 * INSN_COST);
13899
13900 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13901
13902 ins_encode %{
13903 __ ldr($dst$$Register, Address(sp, $src$$disp));
13904 %}
13905
13906 ins_pipe(iload_reg_reg);
13907
13908 %}
13909
13910 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13911
13912 match(Set dst (MoveL2D src));
13913
13914 effect(DEF dst, USE src);
13915
13916 ins_cost(4 * INSN_COST);
13917
13918 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13919
13920 ins_encode %{
13921 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13922 %}
13923
13924 ins_pipe(pipe_class_memory);
13925
13926 %}
13927
13928 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13929
13930 match(Set dst (MoveF2I src));
13931
13932 effect(DEF dst, USE src);
13933
13934 ins_cost(INSN_COST);
13935
13936 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13937
13938 ins_encode %{
13939 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13940 %}
13941
13942 ins_pipe(pipe_class_memory);
13943
13944 %}
13945
13946 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13947
13948 match(Set dst (MoveI2F src));
13949
13950 effect(DEF dst, USE src);
13951
13952 ins_cost(INSN_COST);
13953
13954 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13955
13956 ins_encode %{
13957 __ strw($src$$Register, Address(sp, $dst$$disp));
13958 %}
13959
13960 ins_pipe(istore_reg_reg);
13961
13962 %}
13963
13964 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13965
13966 match(Set dst (MoveD2L src));
13967
13968 effect(DEF dst, USE src);
13969
13970 ins_cost(INSN_COST);
13971
13972 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13973
13974 ins_encode %{
13975 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13976 %}
13977
13978 ins_pipe(pipe_class_memory);
13979
13980 %}
13981
13982 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13983
13984 match(Set dst (MoveL2D src));
13985
13986 effect(DEF dst, USE src);
13987
13988 ins_cost(INSN_COST);
13989
13990 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13991
13992 ins_encode %{
13993 __ str($src$$Register, Address(sp, $dst$$disp));
13994 %}
13995
13996 ins_pipe(istore_reg_reg);
13997
13998 %}
13999
14000 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14001
14002 match(Set dst (MoveF2I src));
14003
14004 effect(DEF dst, USE src);
14005
14006 ins_cost(INSN_COST);
14007
14008 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14009
14010 ins_encode %{
14011 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14012 %}
14013
14014 ins_pipe(fp_f2i);
14015
14016 %}
14017
14018 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14019
14020 match(Set dst (MoveI2F src));
14021
14022 effect(DEF dst, USE src);
14023
14024 ins_cost(INSN_COST);
14025
14026 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14027
14028 ins_encode %{
14029 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14030 %}
14031
14032 ins_pipe(fp_i2f);
14033
14034 %}
14035
14036 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14037
14038 match(Set dst (MoveD2L src));
14039
14040 effect(DEF dst, USE src);
14041
14042 ins_cost(INSN_COST);
14043
14044 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14045
14046 ins_encode %{
14047 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14048 %}
14049
14050 ins_pipe(fp_d2l);
14051
14052 %}
14053
14054 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14055
14056 match(Set dst (MoveL2D src));
14057
14058 effect(DEF dst, USE src);
14059
14060 ins_cost(INSN_COST);
14061
14062 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14063
14064 ins_encode %{
14065 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14066 %}
14067
14068 ins_pipe(fp_l2d);
14069
14070 %}
14071
14072 // ============================================================================
14073 // clearing of an array
14074
14075 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14076 %{
14077 match(Set dummy (ClearArray cnt base));
14078 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14079
14080 ins_cost(4 * INSN_COST);
14081 format %{ "ClearArray $cnt, $base" %}
14082
14083 ins_encode %{
14084 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14085 if (tpc == nullptr) {
14086 ciEnv::current()->record_failure("CodeCache is full");
14087 return;
14088 }
14089 %}
14090
14091 ins_pipe(pipe_class_memory);
14092 %}
14093
14094 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14095 %{
14096 predicate((uint64_t)n->in(2)->get_long()
14097 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14098 match(Set dummy (ClearArray cnt base));
14099 effect(TEMP temp, USE_KILL base, KILL cr);
14100
14101 ins_cost(4 * INSN_COST);
14102 format %{ "ClearArray $cnt, $base" %}
14103
14104 ins_encode %{
14105 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14106 if (tpc == nullptr) {
14107 ciEnv::current()->record_failure("CodeCache is full");
14108 return;
14109 }
14110 %}
14111
14112 ins_pipe(pipe_class_memory);
14113 %}
14114
14115 // ============================================================================
14116 // Overflow Math Instructions
14117
14118 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14119 %{
14120 match(Set cr (OverflowAddI op1 op2));
14121
14122 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14123 ins_cost(INSN_COST);
14124 ins_encode %{
14125 __ cmnw($op1$$Register, $op2$$Register);
14126 %}
14127
14128 ins_pipe(icmp_reg_reg);
14129 %}
14130
14131 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14132 %{
14133 match(Set cr (OverflowAddI op1 op2));
14134
14135 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14136 ins_cost(INSN_COST);
14137 ins_encode %{
14138 __ cmnw($op1$$Register, $op2$$constant);
14139 %}
14140
14141 ins_pipe(icmp_reg_imm);
14142 %}
14143
14144 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14145 %{
14146 match(Set cr (OverflowAddL op1 op2));
14147
14148 format %{ "cmn $op1, $op2\t# overflow check long" %}
14149 ins_cost(INSN_COST);
14150 ins_encode %{
14151 __ cmn($op1$$Register, $op2$$Register);
14152 %}
14153
14154 ins_pipe(icmp_reg_reg);
14155 %}
14156
14157 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14158 %{
14159 match(Set cr (OverflowAddL op1 op2));
14160
14161 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14162 ins_cost(INSN_COST);
14163 ins_encode %{
14164 __ adds(zr, $op1$$Register, $op2$$constant);
14165 %}
14166
14167 ins_pipe(icmp_reg_imm);
14168 %}
14169
14170 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14171 %{
14172 match(Set cr (OverflowSubI op1 op2));
14173
14174 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14175 ins_cost(INSN_COST);
14176 ins_encode %{
14177 __ cmpw($op1$$Register, $op2$$Register);
14178 %}
14179
14180 ins_pipe(icmp_reg_reg);
14181 %}
14182
14183 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14184 %{
14185 match(Set cr (OverflowSubI op1 op2));
14186
14187 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14188 ins_cost(INSN_COST);
14189 ins_encode %{
14190 __ cmpw($op1$$Register, $op2$$constant);
14191 %}
14192
14193 ins_pipe(icmp_reg_imm);
14194 %}
14195
14196 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14197 %{
14198 match(Set cr (OverflowSubL op1 op2));
14199
14200 format %{ "cmp $op1, $op2\t# overflow check long" %}
14201 ins_cost(INSN_COST);
14202 ins_encode %{
14203 __ cmp($op1$$Register, $op2$$Register);
14204 %}
14205
14206 ins_pipe(icmp_reg_reg);
14207 %}
14208
14209 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14210 %{
14211 match(Set cr (OverflowSubL op1 op2));
14212
14213 format %{ "cmp $op1, $op2\t# overflow check long" %}
14214 ins_cost(INSN_COST);
14215 ins_encode %{
14216 __ subs(zr, $op1$$Register, $op2$$constant);
14217 %}
14218
14219 ins_pipe(icmp_reg_imm);
14220 %}
14221
14222 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14223 %{
14224 match(Set cr (OverflowSubI zero op1));
14225
14226 format %{ "cmpw zr, $op1\t# overflow check int" %}
14227 ins_cost(INSN_COST);
14228 ins_encode %{
14229 __ cmpw(zr, $op1$$Register);
14230 %}
14231
14232 ins_pipe(icmp_reg_imm);
14233 %}
14234
14235 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14236 %{
14237 match(Set cr (OverflowSubL zero op1));
14238
14239 format %{ "cmp zr, $op1\t# overflow check long" %}
14240 ins_cost(INSN_COST);
14241 ins_encode %{
14242 __ cmp(zr, $op1$$Register);
14243 %}
14244
14245 ins_pipe(icmp_reg_imm);
14246 %}
14247
14248 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14249 %{
14250 match(Set cr (OverflowMulI op1 op2));
14251
14252 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14253 "cmp rscratch1, rscratch1, sxtw\n\t"
14254 "movw rscratch1, #0x80000000\n\t"
14255 "cselw rscratch1, rscratch1, zr, NE\n\t"
14256 "cmpw rscratch1, #1" %}
14257 ins_cost(5 * INSN_COST);
14258 ins_encode %{
14259 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14260 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14261 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14262 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14263 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14264 %}
14265
14266 ins_pipe(pipe_slow);
14267 %}
14268
14269 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14270 %{
14271 match(If cmp (OverflowMulI op1 op2));
14272 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14273 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14274 effect(USE labl, KILL cr);
14275
14276 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14277 "cmp rscratch1, rscratch1, sxtw\n\t"
14278 "b$cmp $labl" %}
14279 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14280 ins_encode %{
14281 Label* L = $labl$$label;
14282 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14283 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14284 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14285 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14286 %}
14287
14288 ins_pipe(pipe_serial);
14289 %}
14290
14291 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14292 %{
14293 match(Set cr (OverflowMulL op1 op2));
14294
14295 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14296 "smulh rscratch2, $op1, $op2\n\t"
14297 "cmp rscratch2, rscratch1, ASR #63\n\t"
14298 "movw rscratch1, #0x80000000\n\t"
14299 "cselw rscratch1, rscratch1, zr, NE\n\t"
14300 "cmpw rscratch1, #1" %}
14301 ins_cost(6 * INSN_COST);
14302 ins_encode %{
14303 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14304 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14305 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14306 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14307 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14308 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14309 %}
14310
14311 ins_pipe(pipe_slow);
14312 %}
14313
14314 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14315 %{
14316 match(If cmp (OverflowMulL op1 op2));
14317 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14318 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14319 effect(USE labl, KILL cr);
14320
14321 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14322 "smulh rscratch2, $op1, $op2\n\t"
14323 "cmp rscratch2, rscratch1, ASR #63\n\t"
14324 "b$cmp $labl" %}
14325 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14326 ins_encode %{
14327 Label* L = $labl$$label;
14328 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14329 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14330 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14331 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14332 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14333 %}
14334
14335 ins_pipe(pipe_serial);
14336 %}
14337
14338 // ============================================================================
14339 // Compare Instructions
14340
14341 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14342 %{
14343 match(Set cr (CmpI op1 op2));
14344
14345 effect(DEF cr, USE op1, USE op2);
14346
14347 ins_cost(INSN_COST);
14348 format %{ "cmpw $op1, $op2" %}
14349
14350 ins_encode(aarch64_enc_cmpw(op1, op2));
14351
14352 ins_pipe(icmp_reg_reg);
14353 %}
14354
14355 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14356 %{
14357 match(Set cr (CmpI op1 zero));
14358
14359 effect(DEF cr, USE op1);
14360
14361 ins_cost(INSN_COST);
14362 format %{ "cmpw $op1, 0" %}
14363
14364 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14365
14366 ins_pipe(icmp_reg_imm);
14367 %}
14368
14369 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14370 %{
14371 match(Set cr (CmpI op1 op2));
14372
14373 effect(DEF cr, USE op1);
14374
14375 ins_cost(INSN_COST);
14376 format %{ "cmpw $op1, $op2" %}
14377
14378 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14379
14380 ins_pipe(icmp_reg_imm);
14381 %}
14382
14383 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14384 %{
14385 match(Set cr (CmpI op1 op2));
14386
14387 effect(DEF cr, USE op1);
14388
14389 ins_cost(INSN_COST * 2);
14390 format %{ "cmpw $op1, $op2" %}
14391
14392 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14393
14394 ins_pipe(icmp_reg_imm);
14395 %}
14396
14397 // Unsigned compare Instructions; really, same as signed compare
14398 // except it should only be used to feed an If or a CMovI which takes a
14399 // cmpOpU.
14400
14401 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14402 %{
14403 match(Set cr (CmpU op1 op2));
14404
14405 effect(DEF cr, USE op1, USE op2);
14406
14407 ins_cost(INSN_COST);
14408 format %{ "cmpw $op1, $op2\t# unsigned" %}
14409
14410 ins_encode(aarch64_enc_cmpw(op1, op2));
14411
14412 ins_pipe(icmp_reg_reg);
14413 %}
14414
14415 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14416 %{
14417 match(Set cr (CmpU op1 zero));
14418
14419 effect(DEF cr, USE op1);
14420
14421 ins_cost(INSN_COST);
14422 format %{ "cmpw $op1, #0\t# unsigned" %}
14423
14424 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14425
14426 ins_pipe(icmp_reg_imm);
14427 %}
14428
14429 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14430 %{
14431 match(Set cr (CmpU op1 op2));
14432
14433 effect(DEF cr, USE op1);
14434
14435 ins_cost(INSN_COST);
14436 format %{ "cmpw $op1, $op2\t# unsigned" %}
14437
14438 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14439
14440 ins_pipe(icmp_reg_imm);
14441 %}
14442
14443 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14444 %{
14445 match(Set cr (CmpU op1 op2));
14446
14447 effect(DEF cr, USE op1);
14448
14449 ins_cost(INSN_COST * 2);
14450 format %{ "cmpw $op1, $op2\t# unsigned" %}
14451
14452 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14453
14454 ins_pipe(icmp_reg_imm);
14455 %}
14456
14457 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14458 %{
14459 match(Set cr (CmpL op1 op2));
14460
14461 effect(DEF cr, USE op1, USE op2);
14462
14463 ins_cost(INSN_COST);
14464 format %{ "cmp $op1, $op2" %}
14465
14466 ins_encode(aarch64_enc_cmp(op1, op2));
14467
14468 ins_pipe(icmp_reg_reg);
14469 %}
14470
14471 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14472 %{
14473 match(Set cr (CmpL op1 zero));
14474
14475 effect(DEF cr, USE op1);
14476
14477 ins_cost(INSN_COST);
14478 format %{ "tst $op1" %}
14479
14480 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14481
14482 ins_pipe(icmp_reg_imm);
14483 %}
14484
14485 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14486 %{
14487 match(Set cr (CmpL op1 op2));
14488
14489 effect(DEF cr, USE op1);
14490
14491 ins_cost(INSN_COST);
14492 format %{ "cmp $op1, $op2" %}
14493
14494 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14495
14496 ins_pipe(icmp_reg_imm);
14497 %}
14498
14499 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14500 %{
14501 match(Set cr (CmpL op1 op2));
14502
14503 effect(DEF cr, USE op1);
14504
14505 ins_cost(INSN_COST * 2);
14506 format %{ "cmp $op1, $op2" %}
14507
14508 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14509
14510 ins_pipe(icmp_reg_imm);
14511 %}
14512
14513 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14514 %{
14515 match(Set cr (CmpUL op1 op2));
14516
14517 effect(DEF cr, USE op1, USE op2);
14518
14519 ins_cost(INSN_COST);
14520 format %{ "cmp $op1, $op2" %}
14521
14522 ins_encode(aarch64_enc_cmp(op1, op2));
14523
14524 ins_pipe(icmp_reg_reg);
14525 %}
14526
14527 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14528 %{
14529 match(Set cr (CmpUL op1 zero));
14530
14531 effect(DEF cr, USE op1);
14532
14533 ins_cost(INSN_COST);
14534 format %{ "tst $op1" %}
14535
14536 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14537
14538 ins_pipe(icmp_reg_imm);
14539 %}
14540
14541 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14542 %{
14543 match(Set cr (CmpUL op1 op2));
14544
14545 effect(DEF cr, USE op1);
14546
14547 ins_cost(INSN_COST);
14548 format %{ "cmp $op1, $op2" %}
14549
14550 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14551
14552 ins_pipe(icmp_reg_imm);
14553 %}
14554
14555 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14556 %{
14557 match(Set cr (CmpUL op1 op2));
14558
14559 effect(DEF cr, USE op1);
14560
14561 ins_cost(INSN_COST * 2);
14562 format %{ "cmp $op1, $op2" %}
14563
14564 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14565
14566 ins_pipe(icmp_reg_imm);
14567 %}
14568
14569 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14570 %{
14571 match(Set cr (CmpP op1 op2));
14572
14573 effect(DEF cr, USE op1, USE op2);
14574
14575 ins_cost(INSN_COST);
14576 format %{ "cmp $op1, $op2\t // ptr" %}
14577
14578 ins_encode(aarch64_enc_cmpp(op1, op2));
14579
14580 ins_pipe(icmp_reg_reg);
14581 %}
14582
14583 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14584 %{
14585 match(Set cr (CmpN op1 op2));
14586
14587 effect(DEF cr, USE op1, USE op2);
14588
14589 ins_cost(INSN_COST);
14590 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14591
14592 ins_encode(aarch64_enc_cmpn(op1, op2));
14593
14594 ins_pipe(icmp_reg_reg);
14595 %}
14596
14597 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14598 %{
14599 match(Set cr (CmpP op1 zero));
14600
14601 effect(DEF cr, USE op1, USE zero);
14602
14603 ins_cost(INSN_COST);
14604 format %{ "cmp $op1, 0\t // ptr" %}
14605
14606 ins_encode(aarch64_enc_testp(op1));
14607
14608 ins_pipe(icmp_reg_imm);
14609 %}
14610
14611 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14612 %{
14613 match(Set cr (CmpN op1 zero));
14614
14615 effect(DEF cr, USE op1, USE zero);
14616
14617 ins_cost(INSN_COST);
14618 format %{ "cmp $op1, 0\t // compressed ptr" %}
14619
14620 ins_encode(aarch64_enc_testn(op1));
14621
14622 ins_pipe(icmp_reg_imm);
14623 %}
14624
14625 // FP comparisons
14626 //
14627 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14628 // using normal cmpOp. See declaration of rFlagsReg for details.
14629
14630 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14631 %{
14632 match(Set cr (CmpF src1 src2));
14633
14634 ins_cost(3 * INSN_COST);
14635 format %{ "fcmps $src1, $src2" %}
14636
14637 ins_encode %{
14638 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14639 %}
14640
14641 ins_pipe(pipe_class_compare);
14642 %}
14643
14644 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14645 %{
14646 match(Set cr (CmpF src1 src2));
14647
14648 ins_cost(3 * INSN_COST);
14649 format %{ "fcmps $src1, 0.0" %}
14650
14651 ins_encode %{
14652 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14653 %}
14654
14655 ins_pipe(pipe_class_compare);
14656 %}
14657 // FROM HERE
14658
14659 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14660 %{
14661 match(Set cr (CmpD src1 src2));
14662
14663 ins_cost(3 * INSN_COST);
14664 format %{ "fcmpd $src1, $src2" %}
14665
14666 ins_encode %{
14667 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14668 %}
14669
14670 ins_pipe(pipe_class_compare);
14671 %}
14672
14673 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14674 %{
14675 match(Set cr (CmpD src1 src2));
14676
14677 ins_cost(3 * INSN_COST);
14678 format %{ "fcmpd $src1, 0.0" %}
14679
14680 ins_encode %{
14681 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14682 %}
14683
14684 ins_pipe(pipe_class_compare);
14685 %}
14686
14687 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14688 %{
14689 match(Set dst (CmpF3 src1 src2));
14690 effect(KILL cr);
14691
14692 ins_cost(5 * INSN_COST);
14693 format %{ "fcmps $src1, $src2\n\t"
14694 "csinvw($dst, zr, zr, eq\n\t"
14695 "csnegw($dst, $dst, $dst, lt)"
14696 %}
14697
14698 ins_encode %{
14699 Label done;
14700 FloatRegister s1 = as_FloatRegister($src1$$reg);
14701 FloatRegister s2 = as_FloatRegister($src2$$reg);
14702 Register d = as_Register($dst$$reg);
14703 __ fcmps(s1, s2);
14704 // installs 0 if EQ else -1
14705 __ csinvw(d, zr, zr, Assembler::EQ);
14706 // keeps -1 if less or unordered else installs 1
14707 __ csnegw(d, d, d, Assembler::LT);
14708 __ bind(done);
14709 %}
14710
14711 ins_pipe(pipe_class_default);
14712
14713 %}
14714
14715 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14716 %{
14717 match(Set dst (CmpD3 src1 src2));
14718 effect(KILL cr);
14719
14720 ins_cost(5 * INSN_COST);
14721 format %{ "fcmpd $src1, $src2\n\t"
14722 "csinvw($dst, zr, zr, eq\n\t"
14723 "csnegw($dst, $dst, $dst, lt)"
14724 %}
14725
14726 ins_encode %{
14727 Label done;
14728 FloatRegister s1 = as_FloatRegister($src1$$reg);
14729 FloatRegister s2 = as_FloatRegister($src2$$reg);
14730 Register d = as_Register($dst$$reg);
14731 __ fcmpd(s1, s2);
14732 // installs 0 if EQ else -1
14733 __ csinvw(d, zr, zr, Assembler::EQ);
14734 // keeps -1 if less or unordered else installs 1
14735 __ csnegw(d, d, d, Assembler::LT);
14736 __ bind(done);
14737 %}
14738 ins_pipe(pipe_class_default);
14739
14740 %}
14741
14742 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14743 %{
14744 match(Set dst (CmpF3 src1 zero));
14745 effect(KILL cr);
14746
14747 ins_cost(5 * INSN_COST);
14748 format %{ "fcmps $src1, 0.0\n\t"
14749 "csinvw($dst, zr, zr, eq\n\t"
14750 "csnegw($dst, $dst, $dst, lt)"
14751 %}
14752
14753 ins_encode %{
14754 Label done;
14755 FloatRegister s1 = as_FloatRegister($src1$$reg);
14756 Register d = as_Register($dst$$reg);
14757 __ fcmps(s1, 0.0);
14758 // installs 0 if EQ else -1
14759 __ csinvw(d, zr, zr, Assembler::EQ);
14760 // keeps -1 if less or unordered else installs 1
14761 __ csnegw(d, d, d, Assembler::LT);
14762 __ bind(done);
14763 %}
14764
14765 ins_pipe(pipe_class_default);
14766
14767 %}
14768
14769 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14770 %{
14771 match(Set dst (CmpD3 src1 zero));
14772 effect(KILL cr);
14773
14774 ins_cost(5 * INSN_COST);
14775 format %{ "fcmpd $src1, 0.0\n\t"
14776 "csinvw($dst, zr, zr, eq\n\t"
14777 "csnegw($dst, $dst, $dst, lt)"
14778 %}
14779
14780 ins_encode %{
14781 Label done;
14782 FloatRegister s1 = as_FloatRegister($src1$$reg);
14783 Register d = as_Register($dst$$reg);
14784 __ fcmpd(s1, 0.0);
14785 // installs 0 if EQ else -1
14786 __ csinvw(d, zr, zr, Assembler::EQ);
14787 // keeps -1 if less or unordered else installs 1
14788 __ csnegw(d, d, d, Assembler::LT);
14789 __ bind(done);
14790 %}
14791 ins_pipe(pipe_class_default);
14792
14793 %}
14794
14795 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14796 %{
14797 match(Set dst (CmpLTMask p q));
14798 effect(KILL cr);
14799
14800 ins_cost(3 * INSN_COST);
14801
14802 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14803 "csetw $dst, lt\n\t"
14804 "subw $dst, zr, $dst"
14805 %}
14806
14807 ins_encode %{
14808 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14809 __ csetw(as_Register($dst$$reg), Assembler::LT);
14810 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14811 %}
14812
14813 ins_pipe(ialu_reg_reg);
14814 %}
14815
14816 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14817 %{
14818 match(Set dst (CmpLTMask src zero));
14819 effect(KILL cr);
14820
14821 ins_cost(INSN_COST);
14822
14823 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14824
14825 ins_encode %{
14826 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14827 %}
14828
14829 ins_pipe(ialu_reg_shift);
14830 %}
14831
14832 // ============================================================================
14833 // Max and Min
14834
14835 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14836
14837 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14838 %{
14839 effect(DEF cr, USE src);
14840 ins_cost(INSN_COST);
14841 format %{ "cmpw $src, 0" %}
14842
14843 ins_encode %{
14844 __ cmpw($src$$Register, 0);
14845 %}
14846 ins_pipe(icmp_reg_imm);
14847 %}
14848
14849 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14850 %{
14851 match(Set dst (MinI src1 src2));
14852 ins_cost(INSN_COST * 3);
14853
14854 expand %{
14855 rFlagsReg cr;
14856 compI_reg_reg(cr, src1, src2);
14857 cmovI_reg_reg_lt(dst, src1, src2, cr);
14858 %}
14859 %}
14860
14861 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14862 %{
14863 match(Set dst (MaxI src1 src2));
14864 ins_cost(INSN_COST * 3);
14865
14866 expand %{
14867 rFlagsReg cr;
14868 compI_reg_reg(cr, src1, src2);
14869 cmovI_reg_reg_gt(dst, src1, src2, cr);
14870 %}
14871 %}
14872
14873
14874 // ============================================================================
14875 // Branch Instructions
14876
14877 // Direct Branch.
14878 instruct branch(label lbl)
14879 %{
14880 match(Goto);
14881
14882 effect(USE lbl);
14883
14884 ins_cost(BRANCH_COST);
14885 format %{ "b $lbl" %}
14886
14887 ins_encode(aarch64_enc_b(lbl));
14888
14889 ins_pipe(pipe_branch);
14890 %}
14891
14892 // Conditional Near Branch
14893 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14894 %{
14895 // Same match rule as `branchConFar'.
14896 match(If cmp cr);
14897
14898 effect(USE lbl);
14899
14900 ins_cost(BRANCH_COST);
14901 // If set to 1 this indicates that the current instruction is a
14902 // short variant of a long branch. This avoids using this
14903 // instruction in first-pass matching. It will then only be used in
14904 // the `Shorten_branches' pass.
14905 // ins_short_branch(1);
14906 format %{ "b$cmp $lbl" %}
14907
14908 ins_encode(aarch64_enc_br_con(cmp, lbl));
14909
14910 ins_pipe(pipe_branch_cond);
14911 %}
14912
14913 // Conditional Near Branch Unsigned
14914 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14915 %{
14916 // Same match rule as `branchConFar'.
14917 match(If cmp cr);
14918
14919 effect(USE lbl);
14920
14921 ins_cost(BRANCH_COST);
14922 // If set to 1 this indicates that the current instruction is a
14923 // short variant of a long branch. This avoids using this
14924 // instruction in first-pass matching. It will then only be used in
14925 // the `Shorten_branches' pass.
14926 // ins_short_branch(1);
14927 format %{ "b$cmp $lbl\t# unsigned" %}
14928
14929 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14930
14931 ins_pipe(pipe_branch_cond);
14932 %}
14933
14934 // Make use of CBZ and CBNZ. These instructions, as well as being
14935 // shorter than (cmp; branch), have the additional benefit of not
14936 // killing the flags.
14937
14938 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14939 match(If cmp (CmpI op1 op2));
14940 effect(USE labl);
14941
14942 ins_cost(BRANCH_COST);
14943 format %{ "cbw$cmp $op1, $labl" %}
14944 ins_encode %{
14945 Label* L = $labl$$label;
14946 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14947 if (cond == Assembler::EQ)
14948 __ cbzw($op1$$Register, *L);
14949 else
14950 __ cbnzw($op1$$Register, *L);
14951 %}
14952 ins_pipe(pipe_cmp_branch);
14953 %}
14954
14955 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14956 match(If cmp (CmpL op1 op2));
14957 effect(USE labl);
14958
14959 ins_cost(BRANCH_COST);
14960 format %{ "cb$cmp $op1, $labl" %}
14961 ins_encode %{
14962 Label* L = $labl$$label;
14963 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14964 if (cond == Assembler::EQ)
14965 __ cbz($op1$$Register, *L);
14966 else
14967 __ cbnz($op1$$Register, *L);
14968 %}
14969 ins_pipe(pipe_cmp_branch);
14970 %}
14971
14972 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14973 match(If cmp (CmpP op1 op2));
14974 effect(USE labl);
14975
14976 ins_cost(BRANCH_COST);
14977 format %{ "cb$cmp $op1, $labl" %}
14978 ins_encode %{
14979 Label* L = $labl$$label;
14980 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14981 if (cond == Assembler::EQ)
14982 __ cbz($op1$$Register, *L);
14983 else
14984 __ cbnz($op1$$Register, *L);
14985 %}
14986 ins_pipe(pipe_cmp_branch);
14987 %}
14988
14989 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14990 match(If cmp (CmpN op1 op2));
14991 effect(USE labl);
14992
14993 ins_cost(BRANCH_COST);
14994 format %{ "cbw$cmp $op1, $labl" %}
14995 ins_encode %{
14996 Label* L = $labl$$label;
14997 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14998 if (cond == Assembler::EQ)
14999 __ cbzw($op1$$Register, *L);
15000 else
15001 __ cbnzw($op1$$Register, *L);
15002 %}
15003 ins_pipe(pipe_cmp_branch);
15004 %}
15005
15006 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15007 match(If cmp (CmpP (DecodeN oop) zero));
15008 effect(USE labl);
15009
15010 ins_cost(BRANCH_COST);
15011 format %{ "cb$cmp $oop, $labl" %}
15012 ins_encode %{
15013 Label* L = $labl$$label;
15014 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15015 if (cond == Assembler::EQ)
15016 __ cbzw($oop$$Register, *L);
15017 else
15018 __ cbnzw($oop$$Register, *L);
15019 %}
15020 ins_pipe(pipe_cmp_branch);
15021 %}
15022
15023 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15024 match(If cmp (CmpU op1 op2));
15025 effect(USE labl);
15026
15027 ins_cost(BRANCH_COST);
15028 format %{ "cbw$cmp $op1, $labl" %}
15029 ins_encode %{
15030 Label* L = $labl$$label;
15031 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15032 if (cond == Assembler::EQ || cond == Assembler::LS) {
15033 __ cbzw($op1$$Register, *L);
15034 } else {
15035 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15036 __ cbnzw($op1$$Register, *L);
15037 }
15038 %}
15039 ins_pipe(pipe_cmp_branch);
15040 %}
15041
15042 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15043 match(If cmp (CmpUL op1 op2));
15044 effect(USE labl);
15045
15046 ins_cost(BRANCH_COST);
15047 format %{ "cb$cmp $op1, $labl" %}
15048 ins_encode %{
15049 Label* L = $labl$$label;
15050 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15051 if (cond == Assembler::EQ || cond == Assembler::LS) {
15052 __ cbz($op1$$Register, *L);
15053 } else {
15054 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15055 __ cbnz($op1$$Register, *L);
15056 }
15057 %}
15058 ins_pipe(pipe_cmp_branch);
15059 %}
15060
15061 // Test bit and Branch
15062
15063 // Patterns for short (< 32KiB) variants
15064 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15065 match(If cmp (CmpL op1 op2));
15066 effect(USE labl);
15067
15068 ins_cost(BRANCH_COST);
15069 format %{ "cb$cmp $op1, $labl # long" %}
15070 ins_encode %{
15071 Label* L = $labl$$label;
15072 Assembler::Condition cond =
15073 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15074 __ tbr(cond, $op1$$Register, 63, *L);
15075 %}
15076 ins_pipe(pipe_cmp_branch);
15077 ins_short_branch(1);
15078 %}
15079
15080 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15081 match(If cmp (CmpI op1 op2));
15082 effect(USE labl);
15083
15084 ins_cost(BRANCH_COST);
15085 format %{ "cb$cmp $op1, $labl # int" %}
15086 ins_encode %{
15087 Label* L = $labl$$label;
15088 Assembler::Condition cond =
15089 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15090 __ tbr(cond, $op1$$Register, 31, *L);
15091 %}
15092 ins_pipe(pipe_cmp_branch);
15093 ins_short_branch(1);
15094 %}
15095
15096 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15097 match(If cmp (CmpL (AndL op1 op2) op3));
15098 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15099 effect(USE labl);
15100
15101 ins_cost(BRANCH_COST);
15102 format %{ "tb$cmp $op1, $op2, $labl" %}
15103 ins_encode %{
15104 Label* L = $labl$$label;
15105 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15106 int bit = exact_log2_long($op2$$constant);
15107 __ tbr(cond, $op1$$Register, bit, *L);
15108 %}
15109 ins_pipe(pipe_cmp_branch);
15110 ins_short_branch(1);
15111 %}
15112
15113 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15114 match(If cmp (CmpI (AndI op1 op2) op3));
15115 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15116 effect(USE labl);
15117
15118 ins_cost(BRANCH_COST);
15119 format %{ "tb$cmp $op1, $op2, $labl" %}
15120 ins_encode %{
15121 Label* L = $labl$$label;
15122 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15123 int bit = exact_log2((juint)$op2$$constant);
15124 __ tbr(cond, $op1$$Register, bit, *L);
15125 %}
15126 ins_pipe(pipe_cmp_branch);
15127 ins_short_branch(1);
15128 %}
15129
15130 // And far variants
15131 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15132 match(If cmp (CmpL op1 op2));
15133 effect(USE labl);
15134
15135 ins_cost(BRANCH_COST);
15136 format %{ "cb$cmp $op1, $labl # long" %}
15137 ins_encode %{
15138 Label* L = $labl$$label;
15139 Assembler::Condition cond =
15140 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15141 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15142 %}
15143 ins_pipe(pipe_cmp_branch);
15144 %}
15145
15146 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15147 match(If cmp (CmpI op1 op2));
15148 effect(USE labl);
15149
15150 ins_cost(BRANCH_COST);
15151 format %{ "cb$cmp $op1, $labl # int" %}
15152 ins_encode %{
15153 Label* L = $labl$$label;
15154 Assembler::Condition cond =
15155 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15156 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15157 %}
15158 ins_pipe(pipe_cmp_branch);
15159 %}
15160
15161 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15162 match(If cmp (CmpL (AndL op1 op2) op3));
15163 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15164 effect(USE labl);
15165
15166 ins_cost(BRANCH_COST);
15167 format %{ "tb$cmp $op1, $op2, $labl" %}
15168 ins_encode %{
15169 Label* L = $labl$$label;
15170 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15171 int bit = exact_log2_long($op2$$constant);
15172 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15173 %}
15174 ins_pipe(pipe_cmp_branch);
15175 %}
15176
15177 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15178 match(If cmp (CmpI (AndI op1 op2) op3));
15179 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15180 effect(USE labl);
15181
15182 ins_cost(BRANCH_COST);
15183 format %{ "tb$cmp $op1, $op2, $labl" %}
15184 ins_encode %{
15185 Label* L = $labl$$label;
15186 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15187 int bit = exact_log2((juint)$op2$$constant);
15188 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15189 %}
15190 ins_pipe(pipe_cmp_branch);
15191 %}
15192
15193 // Test bits
15194
15195 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15196 match(Set cr (CmpL (AndL op1 op2) op3));
15197 predicate(Assembler::operand_valid_for_logical_immediate
15198 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15199
15200 ins_cost(INSN_COST);
15201 format %{ "tst $op1, $op2 # long" %}
15202 ins_encode %{
15203 __ tst($op1$$Register, $op2$$constant);
15204 %}
15205 ins_pipe(ialu_reg_reg);
15206 %}
15207
15208 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15209 match(Set cr (CmpI (AndI op1 op2) op3));
15210 predicate(Assembler::operand_valid_for_logical_immediate
15211 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15212
15213 ins_cost(INSN_COST);
15214 format %{ "tst $op1, $op2 # int" %}
15215 ins_encode %{
15216 __ tstw($op1$$Register, $op2$$constant);
15217 %}
15218 ins_pipe(ialu_reg_reg);
15219 %}
15220
15221 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15222 match(Set cr (CmpL (AndL op1 op2) op3));
15223
15224 ins_cost(INSN_COST);
15225 format %{ "tst $op1, $op2 # long" %}
15226 ins_encode %{
15227 __ tst($op1$$Register, $op2$$Register);
15228 %}
15229 ins_pipe(ialu_reg_reg);
15230 %}
15231
15232 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15233 match(Set cr (CmpI (AndI op1 op2) op3));
15234
15235 ins_cost(INSN_COST);
15236 format %{ "tstw $op1, $op2 # int" %}
15237 ins_encode %{
15238 __ tstw($op1$$Register, $op2$$Register);
15239 %}
15240 ins_pipe(ialu_reg_reg);
15241 %}
15242
15243
15244 // Conditional Far Branch
15245 // Conditional Far Branch Unsigned
15246 // TODO: fixme
15247
15248 // counted loop end branch near
15249 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15250 %{
15251 match(CountedLoopEnd cmp cr);
15252
15253 effect(USE lbl);
15254
15255 ins_cost(BRANCH_COST);
15256 // short variant.
15257 // ins_short_branch(1);
15258 format %{ "b$cmp $lbl \t// counted loop end" %}
15259
15260 ins_encode(aarch64_enc_br_con(cmp, lbl));
15261
15262 ins_pipe(pipe_branch);
15263 %}
15264
15265 // counted loop end branch far
15266 // TODO: fixme
15267
15268 // ============================================================================
15269 // inlined locking and unlocking
15270
15271 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15272 %{
15273 match(Set cr (FastLock object box));
15274 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15275
15276 ins_cost(5 * INSN_COST);
15277 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15278
15279 ins_encode %{
15280 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15281 %}
15282
15283 ins_pipe(pipe_serial);
15284 %}
15285
15286 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15287 %{
15288 match(Set cr (FastUnlock object box));
15289 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15290
15291 ins_cost(5 * INSN_COST);
15292 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15293
15294 ins_encode %{
15295 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15296 %}
15297
15298 ins_pipe(pipe_serial);
15299 %}
15300
15301 // ============================================================================
15302 // Safepoint Instructions
15303
15304 // TODO
15305 // provide a near and far version of this code
15306
15307 instruct safePoint(rFlagsReg cr, iRegP poll)
15308 %{
15309 match(SafePoint poll);
15310 effect(KILL cr);
15311
15312 format %{
15313 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15314 %}
15315 ins_encode %{
15316 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15317 %}
15318 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15319 %}
15320
15321
15322 // ============================================================================
15323 // Procedure Call/Return Instructions
15324
15325 // Call Java Static Instruction
15326
15327 instruct CallStaticJavaDirect(method meth)
15328 %{
15329 match(CallStaticJava);
15330
15331 effect(USE meth);
15332
15333 ins_cost(CALL_COST);
15334
15335 format %{ "call,static $meth \t// ==> " %}
15336
15337 ins_encode(aarch64_enc_java_static_call(meth),
15338 aarch64_enc_call_epilog);
15339
15340 ins_pipe(pipe_class_call);
15341 %}
15342
15343 // TO HERE
15344
15345 // Call Java Dynamic Instruction
15346 instruct CallDynamicJavaDirect(method meth)
15347 %{
15348 match(CallDynamicJava);
15349
15350 effect(USE meth);
15351
15352 ins_cost(CALL_COST);
15353
15354 format %{ "CALL,dynamic $meth \t// ==> " %}
15355
15356 ins_encode(aarch64_enc_java_dynamic_call(meth),
15357 aarch64_enc_call_epilog);
15358
15359 ins_pipe(pipe_class_call);
15360 %}
15361
15362 // Call Runtime Instruction
15363
15364 instruct CallRuntimeDirect(method meth)
15365 %{
15366 match(CallRuntime);
15367
15368 effect(USE meth);
15369
15370 ins_cost(CALL_COST);
15371
15372 format %{ "CALL, runtime $meth" %}
15373
15374 ins_encode( aarch64_enc_java_to_runtime(meth) );
15375
15376 ins_pipe(pipe_class_call);
15377 %}
15378
15379 // Call Runtime Instruction
15380
15381 instruct CallLeafDirect(method meth)
15382 %{
15383 match(CallLeaf);
15384
15385 effect(USE meth);
15386
15387 ins_cost(CALL_COST);
15388
15389 format %{ "CALL, runtime leaf $meth" %}
15390
15391 ins_encode( aarch64_enc_java_to_runtime(meth) );
15392
15393 ins_pipe(pipe_class_call);
15394 %}
15395
15396 // Call Runtime Instruction without safepoint and with vector arguments
15397 instruct CallLeafDirectVector(method meth)
15398 %{
15399 match(CallLeafVector);
15400
15401 effect(USE meth);
15402
15403 ins_cost(CALL_COST);
15404
15405 format %{ "CALL, runtime leaf vector $meth" %}
15406
15407 ins_encode(aarch64_enc_java_to_runtime(meth));
15408
15409 ins_pipe(pipe_class_call);
15410 %}
15411
15412 // Call Runtime Instruction
15413
15414 instruct CallLeafNoFPDirect(method meth)
15415 %{
15416 match(CallLeafNoFP);
15417
15418 effect(USE meth);
15419
15420 ins_cost(CALL_COST);
15421
15422 format %{ "CALL, runtime leaf nofp $meth" %}
15423
15424 ins_encode( aarch64_enc_java_to_runtime(meth) );
15425
15426 ins_pipe(pipe_class_call);
15427 %}
15428
15429 // Tail Call; Jump from runtime stub to Java code.
15430 // Also known as an 'interprocedural jump'.
15431 // Target of jump will eventually return to caller.
15432 // TailJump below removes the return address.
15433 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15434 // emitted just above the TailCall which has reset rfp to the caller state.
15435 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15436 %{
15437 match(TailCall jump_target method_ptr);
15438
15439 ins_cost(CALL_COST);
15440
15441 format %{ "br $jump_target\t# $method_ptr holds method" %}
15442
15443 ins_encode(aarch64_enc_tail_call(jump_target));
15444
15445 ins_pipe(pipe_class_call);
15446 %}
15447
15448 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15449 %{
15450 match(TailJump jump_target ex_oop);
15451
15452 ins_cost(CALL_COST);
15453
15454 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15455
15456 ins_encode(aarch64_enc_tail_jmp(jump_target));
15457
15458 ins_pipe(pipe_class_call);
15459 %}
15460
15461 // Forward exception.
15462 instruct ForwardExceptionjmp()
15463 %{
15464 match(ForwardException);
15465 ins_cost(CALL_COST);
15466
15467 format %{ "b forward_exception_stub" %}
15468 ins_encode %{
15469 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15470 %}
15471 ins_pipe(pipe_class_call);
15472 %}
15473
15474 // Create exception oop: created by stack-crawling runtime code.
15475 // Created exception is now available to this handler, and is setup
15476 // just prior to jumping to this handler. No code emitted.
15477 // TODO check
15478 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15479 instruct CreateException(iRegP_R0 ex_oop)
15480 %{
15481 match(Set ex_oop (CreateEx));
15482
15483 format %{ " -- \t// exception oop; no code emitted" %}
15484
15485 size(0);
15486
15487 ins_encode( /*empty*/ );
15488
15489 ins_pipe(pipe_class_empty);
15490 %}
15491
15492 // Rethrow exception: The exception oop will come in the first
15493 // argument position. Then JUMP (not call) to the rethrow stub code.
15494 instruct RethrowException() %{
15495 match(Rethrow);
15496 ins_cost(CALL_COST);
15497
15498 format %{ "b rethrow_stub" %}
15499
15500 ins_encode( aarch64_enc_rethrow() );
15501
15502 ins_pipe(pipe_class_call);
15503 %}
15504
15505
15506 // Return Instruction
15507 // epilog node loads ret address into lr as part of frame pop
15508 instruct Ret()
15509 %{
15510 match(Return);
15511
15512 format %{ "ret\t// return register" %}
15513
15514 ins_encode( aarch64_enc_ret() );
15515
15516 ins_pipe(pipe_branch);
15517 %}
15518
15519 // Die now.
15520 instruct ShouldNotReachHere() %{
15521 match(Halt);
15522
15523 ins_cost(CALL_COST);
15524 format %{ "ShouldNotReachHere" %}
15525
15526 ins_encode %{
15527 if (is_reachable()) {
15528 const char* str = __ code_string(_halt_reason);
15529 __ stop(str);
15530 }
15531 %}
15532
15533 ins_pipe(pipe_class_default);
15534 %}
15535
15536 // ============================================================================
15537 // Partial Subtype Check
15538 //
15539 // superklass array for an instance of the superklass. Set a hidden
15540 // internal cache on a hit (cache is checked with exposed code in
15541 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15542 // encoding ALSO sets flags.
15543
15544 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15545 %{
15546 match(Set result (PartialSubtypeCheck sub super));
15547 predicate(!UseSecondarySupersTable);
15548 effect(KILL cr, KILL temp);
15549
15550 ins_cost(20 * INSN_COST); // slightly larger than the next version
15551 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15552
15553 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15554
15555 opcode(0x1); // Force zero of result reg on hit
15556
15557 ins_pipe(pipe_class_memory);
15558 %}
15559
15560 // Two versions of partialSubtypeCheck, both used when we need to
15561 // search for a super class in the secondary supers array. The first
15562 // is used when we don't know _a priori_ the class being searched
15563 // for. The second, far more common, is used when we do know: this is
15564 // used for instanceof, checkcast, and any case where C2 can determine
15565 // it by constant propagation.
15566
15567 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15568 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15569 rFlagsReg cr)
15570 %{
15571 match(Set result (PartialSubtypeCheck sub super));
15572 predicate(UseSecondarySupersTable);
15573 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15574
15575 ins_cost(10 * INSN_COST); // slightly larger than the next version
15576 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15577
15578 ins_encode %{
15579 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15580 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15581 $vtemp$$FloatRegister,
15582 $result$$Register, /*L_success*/nullptr);
15583 %}
15584
15585 ins_pipe(pipe_class_memory);
15586 %}
15587
15588 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15589 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15590 rFlagsReg cr)
15591 %{
15592 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15593 predicate(UseSecondarySupersTable);
15594 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15595
15596 ins_cost(5 * INSN_COST); // smaller than the next version
15597 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15598
15599 ins_encode %{
15600 bool success = false;
15601 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15602 if (InlineSecondarySupersTest) {
15603 success =
15604 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15605 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15606 $vtemp$$FloatRegister,
15607 $result$$Register,
15608 super_klass_slot);
15609 } else {
15610 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15611 success = (call != nullptr);
15612 }
15613 if (!success) {
15614 ciEnv::current()->record_failure("CodeCache is full");
15615 return;
15616 }
15617 %}
15618
15619 ins_pipe(pipe_class_memory);
15620 %}
15621
15622 // Intrisics for String.compareTo()
15623
15624 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15625 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15626 %{
15627 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15628 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15629 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15630
15631 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15632 ins_encode %{
15633 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15634 __ string_compare($str1$$Register, $str2$$Register,
15635 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15636 $tmp1$$Register, $tmp2$$Register,
15637 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15638 %}
15639 ins_pipe(pipe_class_memory);
15640 %}
15641
15642 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15643 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15644 %{
15645 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15646 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15647 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15648
15649 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15650 ins_encode %{
15651 __ string_compare($str1$$Register, $str2$$Register,
15652 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15653 $tmp1$$Register, $tmp2$$Register,
15654 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15655 %}
15656 ins_pipe(pipe_class_memory);
15657 %}
15658
15659 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15660 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15661 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15662 %{
15663 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15664 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15665 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15666 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15667
15668 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15669 ins_encode %{
15670 __ string_compare($str1$$Register, $str2$$Register,
15671 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15672 $tmp1$$Register, $tmp2$$Register,
15673 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15674 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15675 %}
15676 ins_pipe(pipe_class_memory);
15677 %}
15678
15679 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15680 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15681 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15682 %{
15683 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15684 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15685 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15686 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15687
15688 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15689 ins_encode %{
15690 __ string_compare($str1$$Register, $str2$$Register,
15691 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15692 $tmp1$$Register, $tmp2$$Register,
15693 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15694 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15695 %}
15696 ins_pipe(pipe_class_memory);
15697 %}
15698
15699 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15700 // these string_compare variants as NEON register type for convenience so that the prototype of
15701 // string_compare can be shared with all variants.
15702
15703 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15704 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15705 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15706 pRegGov_P1 pgtmp2, rFlagsReg cr)
15707 %{
15708 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15709 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15710 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15711 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15712
15713 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15714 ins_encode %{
15715 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15716 __ string_compare($str1$$Register, $str2$$Register,
15717 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15718 $tmp1$$Register, $tmp2$$Register,
15719 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15720 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15721 StrIntrinsicNode::LL);
15722 %}
15723 ins_pipe(pipe_class_memory);
15724 %}
15725
15726 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15727 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15728 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15729 pRegGov_P1 pgtmp2, rFlagsReg cr)
15730 %{
15731 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15732 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15733 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15734 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15735
15736 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15737 ins_encode %{
15738 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15739 __ string_compare($str1$$Register, $str2$$Register,
15740 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15741 $tmp1$$Register, $tmp2$$Register,
15742 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15743 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15744 StrIntrinsicNode::LU);
15745 %}
15746 ins_pipe(pipe_class_memory);
15747 %}
15748
15749 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15750 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15751 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15752 pRegGov_P1 pgtmp2, rFlagsReg cr)
15753 %{
15754 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15755 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15756 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15757 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15758
15759 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15760 ins_encode %{
15761 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15762 __ string_compare($str1$$Register, $str2$$Register,
15763 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15764 $tmp1$$Register, $tmp2$$Register,
15765 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15766 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15767 StrIntrinsicNode::UL);
15768 %}
15769 ins_pipe(pipe_class_memory);
15770 %}
15771
15772 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15773 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15774 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15775 pRegGov_P1 pgtmp2, rFlagsReg cr)
15776 %{
15777 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15778 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15779 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15780 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15781
15782 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15783 ins_encode %{
15784 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15785 __ string_compare($str1$$Register, $str2$$Register,
15786 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15787 $tmp1$$Register, $tmp2$$Register,
15788 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15789 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15790 StrIntrinsicNode::UU);
15791 %}
15792 ins_pipe(pipe_class_memory);
15793 %}
15794
15795 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15796 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15797 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15798 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15799 %{
15800 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15801 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15802 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15803 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15804 TEMP vtmp0, TEMP vtmp1, KILL cr);
15805 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15806 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15807
15808 ins_encode %{
15809 __ string_indexof($str1$$Register, $str2$$Register,
15810 $cnt1$$Register, $cnt2$$Register,
15811 $tmp1$$Register, $tmp2$$Register,
15812 $tmp3$$Register, $tmp4$$Register,
15813 $tmp5$$Register, $tmp6$$Register,
15814 -1, $result$$Register, StrIntrinsicNode::UU);
15815 %}
15816 ins_pipe(pipe_class_memory);
15817 %}
15818
15819 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15820 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15821 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15822 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15823 %{
15824 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15825 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15826 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15827 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15828 TEMP vtmp0, TEMP vtmp1, KILL cr);
15829 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15830 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15831
15832 ins_encode %{
15833 __ string_indexof($str1$$Register, $str2$$Register,
15834 $cnt1$$Register, $cnt2$$Register,
15835 $tmp1$$Register, $tmp2$$Register,
15836 $tmp3$$Register, $tmp4$$Register,
15837 $tmp5$$Register, $tmp6$$Register,
15838 -1, $result$$Register, StrIntrinsicNode::LL);
15839 %}
15840 ins_pipe(pipe_class_memory);
15841 %}
15842
15843 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15844 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15845 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15846 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15847 %{
15848 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15849 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15850 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15851 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15852 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15853 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15854 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15855
15856 ins_encode %{
15857 __ string_indexof($str1$$Register, $str2$$Register,
15858 $cnt1$$Register, $cnt2$$Register,
15859 $tmp1$$Register, $tmp2$$Register,
15860 $tmp3$$Register, $tmp4$$Register,
15861 $tmp5$$Register, $tmp6$$Register,
15862 -1, $result$$Register, StrIntrinsicNode::UL);
15863 %}
15864 ins_pipe(pipe_class_memory);
15865 %}
15866
15867 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15868 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15869 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15870 %{
15871 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15872 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15873 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15874 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15875 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15876 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15877
15878 ins_encode %{
15879 int icnt2 = (int)$int_cnt2$$constant;
15880 __ string_indexof($str1$$Register, $str2$$Register,
15881 $cnt1$$Register, zr,
15882 $tmp1$$Register, $tmp2$$Register,
15883 $tmp3$$Register, $tmp4$$Register, zr, zr,
15884 icnt2, $result$$Register, StrIntrinsicNode::UU);
15885 %}
15886 ins_pipe(pipe_class_memory);
15887 %}
15888
15889 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15890 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15891 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15892 %{
15893 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15894 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15895 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15896 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15897 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15898 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15899
15900 ins_encode %{
15901 int icnt2 = (int)$int_cnt2$$constant;
15902 __ string_indexof($str1$$Register, $str2$$Register,
15903 $cnt1$$Register, zr,
15904 $tmp1$$Register, $tmp2$$Register,
15905 $tmp3$$Register, $tmp4$$Register, zr, zr,
15906 icnt2, $result$$Register, StrIntrinsicNode::LL);
15907 %}
15908 ins_pipe(pipe_class_memory);
15909 %}
15910
15911 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15912 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15913 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15914 %{
15915 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15916 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15917 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15918 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15919 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15920 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15921
15922 ins_encode %{
15923 int icnt2 = (int)$int_cnt2$$constant;
15924 __ string_indexof($str1$$Register, $str2$$Register,
15925 $cnt1$$Register, zr,
15926 $tmp1$$Register, $tmp2$$Register,
15927 $tmp3$$Register, $tmp4$$Register, zr, zr,
15928 icnt2, $result$$Register, StrIntrinsicNode::UL);
15929 %}
15930 ins_pipe(pipe_class_memory);
15931 %}
15932
15933 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15934 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15935 iRegINoSp tmp3, rFlagsReg cr)
15936 %{
15937 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15938 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15939 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15940 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15941
15942 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15943
15944 ins_encode %{
15945 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15946 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15947 $tmp3$$Register);
15948 %}
15949 ins_pipe(pipe_class_memory);
15950 %}
15951
15952 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15953 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15954 iRegINoSp tmp3, rFlagsReg cr)
15955 %{
15956 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15957 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15958 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15959 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15960
15961 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15962
15963 ins_encode %{
15964 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15965 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15966 $tmp3$$Register);
15967 %}
15968 ins_pipe(pipe_class_memory);
15969 %}
15970
15971 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15972 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15973 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15974 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15975 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15976 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15977 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15978 ins_encode %{
15979 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15980 $result$$Register, $ztmp1$$FloatRegister,
15981 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15982 $ptmp$$PRegister, true /* isL */);
15983 %}
15984 ins_pipe(pipe_class_memory);
15985 %}
15986
15987 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15988 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15989 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15990 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15991 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15992 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15993 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15994 ins_encode %{
15995 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15996 $result$$Register, $ztmp1$$FloatRegister,
15997 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15998 $ptmp$$PRegister, false /* isL */);
15999 %}
16000 ins_pipe(pipe_class_memory);
16001 %}
16002
16003 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16004 iRegI_R0 result, rFlagsReg cr)
16005 %{
16006 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16007 match(Set result (StrEquals (Binary str1 str2) cnt));
16008 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16009
16010 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16011 ins_encode %{
16012 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16013 __ string_equals($str1$$Register, $str2$$Register,
16014 $result$$Register, $cnt$$Register);
16015 %}
16016 ins_pipe(pipe_class_memory);
16017 %}
16018
16019 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16020 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16021 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16022 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16023 iRegP_R10 tmp, rFlagsReg cr)
16024 %{
16025 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16026 match(Set result (AryEq ary1 ary2));
16027 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16028 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16029 TEMP vtmp6, TEMP vtmp7, KILL cr);
16030
16031 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16032 ins_encode %{
16033 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16034 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16035 $result$$Register, $tmp$$Register, 1);
16036 if (tpc == nullptr) {
16037 ciEnv::current()->record_failure("CodeCache is full");
16038 return;
16039 }
16040 %}
16041 ins_pipe(pipe_class_memory);
16042 %}
16043
16044 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16045 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16046 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16047 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16048 iRegP_R10 tmp, rFlagsReg cr)
16049 %{
16050 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16051 match(Set result (AryEq ary1 ary2));
16052 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16053 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16054 TEMP vtmp6, TEMP vtmp7, KILL cr);
16055
16056 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16057 ins_encode %{
16058 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16059 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16060 $result$$Register, $tmp$$Register, 2);
16061 if (tpc == nullptr) {
16062 ciEnv::current()->record_failure("CodeCache is full");
16063 return;
16064 }
16065 %}
16066 ins_pipe(pipe_class_memory);
16067 %}
16068
16069 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16070 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16071 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16072 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16073 %{
16074 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16075 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16076 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16077
16078 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16079 ins_encode %{
16080 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16081 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16082 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16083 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16084 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16085 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16086 (BasicType)$basic_type$$constant);
16087 if (tpc == nullptr) {
16088 ciEnv::current()->record_failure("CodeCache is full");
16089 return;
16090 }
16091 %}
16092 ins_pipe(pipe_class_memory);
16093 %}
16094
16095 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16096 %{
16097 match(Set result (CountPositives ary1 len));
16098 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16099 format %{ "count positives byte[] $ary1,$len -> $result" %}
16100 ins_encode %{
16101 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16102 if (tpc == nullptr) {
16103 ciEnv::current()->record_failure("CodeCache is full");
16104 return;
16105 }
16106 %}
16107 ins_pipe( pipe_slow );
16108 %}
16109
16110 // fast char[] to byte[] compression
16111 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16112 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16113 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16114 iRegI_R0 result, rFlagsReg cr)
16115 %{
16116 match(Set result (StrCompressedCopy src (Binary dst len)));
16117 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16118 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16119
16120 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16121 ins_encode %{
16122 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16123 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16124 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16125 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16126 %}
16127 ins_pipe(pipe_slow);
16128 %}
16129
16130 // fast byte[] to char[] inflation
16131 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16132 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16133 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16134 %{
16135 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16136 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16137 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16138 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16139
16140 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16141 ins_encode %{
16142 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16143 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16144 $vtmp2$$FloatRegister, $tmp$$Register);
16145 if (tpc == nullptr) {
16146 ciEnv::current()->record_failure("CodeCache is full");
16147 return;
16148 }
16149 %}
16150 ins_pipe(pipe_class_memory);
16151 %}
16152
16153 // encode char[] to byte[] in ISO_8859_1
16154 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16155 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16156 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16157 iRegI_R0 result, rFlagsReg cr)
16158 %{
16159 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16160 match(Set result (EncodeISOArray src (Binary dst len)));
16161 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16162 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16163
16164 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16165 ins_encode %{
16166 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16167 $result$$Register, false,
16168 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16169 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16170 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16171 %}
16172 ins_pipe(pipe_class_memory);
16173 %}
16174
16175 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16176 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16177 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16178 iRegI_R0 result, rFlagsReg cr)
16179 %{
16180 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16181 match(Set result (EncodeISOArray src (Binary dst len)));
16182 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16183 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16184
16185 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16186 ins_encode %{
16187 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16188 $result$$Register, true,
16189 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16190 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16191 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16192 %}
16193 ins_pipe(pipe_class_memory);
16194 %}
16195
16196 //----------------------------- CompressBits/ExpandBits ------------------------
16197
16198 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16199 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16200 match(Set dst (CompressBits src mask));
16201 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16202 format %{ "mov $tsrc, $src\n\t"
16203 "mov $tmask, $mask\n\t"
16204 "bext $tdst, $tsrc, $tmask\n\t"
16205 "mov $dst, $tdst"
16206 %}
16207 ins_encode %{
16208 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16209 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16210 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16211 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16212 %}
16213 ins_pipe(pipe_slow);
16214 %}
16215
16216 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16217 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16218 match(Set dst (CompressBits (LoadI mem) mask));
16219 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16220 format %{ "ldrs $tsrc, $mem\n\t"
16221 "ldrs $tmask, $mask\n\t"
16222 "bext $tdst, $tsrc, $tmask\n\t"
16223 "mov $dst, $tdst"
16224 %}
16225 ins_encode %{
16226 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16227 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16228 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16229 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16230 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16231 %}
16232 ins_pipe(pipe_slow);
16233 %}
16234
16235 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16236 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16237 match(Set dst (CompressBits src mask));
16238 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16239 format %{ "mov $tsrc, $src\n\t"
16240 "mov $tmask, $mask\n\t"
16241 "bext $tdst, $tsrc, $tmask\n\t"
16242 "mov $dst, $tdst"
16243 %}
16244 ins_encode %{
16245 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16246 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16247 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16248 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16249 %}
16250 ins_pipe(pipe_slow);
16251 %}
16252
16253 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16254 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16255 match(Set dst (CompressBits (LoadL mem) mask));
16256 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16257 format %{ "ldrd $tsrc, $mem\n\t"
16258 "ldrd $tmask, $mask\n\t"
16259 "bext $tdst, $tsrc, $tmask\n\t"
16260 "mov $dst, $tdst"
16261 %}
16262 ins_encode %{
16263 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16264 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16265 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16266 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16267 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16268 %}
16269 ins_pipe(pipe_slow);
16270 %}
16271
16272 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16273 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16274 match(Set dst (ExpandBits src mask));
16275 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16276 format %{ "mov $tsrc, $src\n\t"
16277 "mov $tmask, $mask\n\t"
16278 "bdep $tdst, $tsrc, $tmask\n\t"
16279 "mov $dst, $tdst"
16280 %}
16281 ins_encode %{
16282 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16283 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16284 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16285 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16286 %}
16287 ins_pipe(pipe_slow);
16288 %}
16289
16290 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16291 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16292 match(Set dst (ExpandBits (LoadI mem) mask));
16293 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16294 format %{ "ldrs $tsrc, $mem\n\t"
16295 "ldrs $tmask, $mask\n\t"
16296 "bdep $tdst, $tsrc, $tmask\n\t"
16297 "mov $dst, $tdst"
16298 %}
16299 ins_encode %{
16300 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16301 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16302 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16303 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16304 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16305 %}
16306 ins_pipe(pipe_slow);
16307 %}
16308
16309 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16310 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16311 match(Set dst (ExpandBits src mask));
16312 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16313 format %{ "mov $tsrc, $src\n\t"
16314 "mov $tmask, $mask\n\t"
16315 "bdep $tdst, $tsrc, $tmask\n\t"
16316 "mov $dst, $tdst"
16317 %}
16318 ins_encode %{
16319 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16320 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16321 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16322 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16323 %}
16324 ins_pipe(pipe_slow);
16325 %}
16326
16327
16328 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16329 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16330 match(Set dst (ExpandBits (LoadL mem) mask));
16331 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16332 format %{ "ldrd $tsrc, $mem\n\t"
16333 "ldrd $tmask, $mask\n\t"
16334 "bdep $tdst, $tsrc, $tmask\n\t"
16335 "mov $dst, $tdst"
16336 %}
16337 ins_encode %{
16338 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16339 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16340 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16341 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16342 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16343 %}
16344 ins_pipe(pipe_slow);
16345 %}
16346
16347 //----------------------------- Reinterpret ----------------------------------
16348 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16349 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16350 match(Set dst (ReinterpretHF2S src));
16351 format %{ "reinterpretHF2S $dst, $src" %}
16352 ins_encode %{
16353 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16354 %}
16355 ins_pipe(pipe_slow);
16356 %}
16357
16358 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16359 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16360 match(Set dst (ReinterpretS2HF src));
16361 format %{ "reinterpretS2HF $dst, $src" %}
16362 ins_encode %{
16363 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16364 %}
16365 ins_pipe(pipe_slow);
16366 %}
16367
16368 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16369 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16370 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16371 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16372 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16373 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16374 // can be omitted in this pattern, resulting in -
16375 // fcvt $dst, $src // Convert float to half-precision float
16376 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16377 %{
16378 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16379 format %{ "convF2HFAndS2HF $dst, $src" %}
16380 ins_encode %{
16381 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16382 %}
16383 ins_pipe(pipe_slow);
16384 %}
16385
16386 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16387 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16388 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16389 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16390 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16391 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16392 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16393 // resulting in -
16394 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16395 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16396 %{
16397 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16398 format %{ "convHF2SAndHF2F $dst, $src" %}
16399 ins_encode %{
16400 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16401 %}
16402 ins_pipe(pipe_slow);
16403 %}
16404
16405 // ============================================================================
16406 // This name is KNOWN by the ADLC and cannot be changed.
16407 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16408 // for this guy.
16409 instruct tlsLoadP(thread_RegP dst)
16410 %{
16411 match(Set dst (ThreadLocal));
16412
16413 ins_cost(0);
16414
16415 format %{ " -- \t// $dst=Thread::current(), empty" %}
16416
16417 size(0);
16418
16419 ins_encode( /*empty*/ );
16420
16421 ins_pipe(pipe_class_empty);
16422 %}
16423
16424 //----------PEEPHOLE RULES-----------------------------------------------------
16425 // These must follow all instruction definitions as they use the names
16426 // defined in the instructions definitions.
16427 //
16428 // peepmatch ( root_instr_name [preceding_instruction]* );
16429 //
16430 // peepconstraint %{
16431 // (instruction_number.operand_name relational_op instruction_number.operand_name
16432 // [, ...] );
16433 // // instruction numbers are zero-based using left to right order in peepmatch
16434 //
16435 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16436 // // provide an instruction_number.operand_name for each operand that appears
16437 // // in the replacement instruction's match rule
16438 //
16439 // ---------VM FLAGS---------------------------------------------------------
16440 //
16441 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16442 //
16443 // Each peephole rule is given an identifying number starting with zero and
16444 // increasing by one in the order seen by the parser. An individual peephole
16445 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16446 // on the command-line.
16447 //
16448 // ---------CURRENT LIMITATIONS----------------------------------------------
16449 //
16450 // Only match adjacent instructions in same basic block
16451 // Only equality constraints
16452 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16453 // Only one replacement instruction
16454 //
16455 // ---------EXAMPLE----------------------------------------------------------
16456 //
16457 // // pertinent parts of existing instructions in architecture description
16458 // instruct movI(iRegINoSp dst, iRegI src)
16459 // %{
16460 // match(Set dst (CopyI src));
16461 // %}
16462 //
16463 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16464 // %{
16465 // match(Set dst (AddI dst src));
16466 // effect(KILL cr);
16467 // %}
16468 //
16469 // // Change (inc mov) to lea
16470 // peephole %{
16471 // // increment preceded by register-register move
16472 // peepmatch ( incI_iReg movI );
16473 // // require that the destination register of the increment
16474 // // match the destination register of the move
16475 // peepconstraint ( 0.dst == 1.dst );
16476 // // construct a replacement instruction that sets
16477 // // the destination to ( move's source register + one )
16478 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16479 // %}
16480 //
16481
16482 // Implementation no longer uses movX instructions since
16483 // machine-independent system no longer uses CopyX nodes.
16484 //
16485 // peephole
16486 // %{
16487 // peepmatch (incI_iReg movI);
16488 // peepconstraint (0.dst == 1.dst);
16489 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16490 // %}
16491
16492 // peephole
16493 // %{
16494 // peepmatch (decI_iReg movI);
16495 // peepconstraint (0.dst == 1.dst);
16496 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16497 // %}
16498
16499 // peephole
16500 // %{
16501 // peepmatch (addI_iReg_imm movI);
16502 // peepconstraint (0.dst == 1.dst);
16503 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16504 // %}
16505
16506 // peephole
16507 // %{
16508 // peepmatch (incL_iReg movL);
16509 // peepconstraint (0.dst == 1.dst);
16510 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16511 // %}
16512
16513 // peephole
16514 // %{
16515 // peepmatch (decL_iReg movL);
16516 // peepconstraint (0.dst == 1.dst);
16517 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16518 // %}
16519
16520 // peephole
16521 // %{
16522 // peepmatch (addL_iReg_imm movL);
16523 // peepconstraint (0.dst == 1.dst);
16524 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16525 // %}
16526
16527 // peephole
16528 // %{
16529 // peepmatch (addP_iReg_imm movP);
16530 // peepconstraint (0.dst == 1.dst);
16531 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16532 // %}
16533
16534 // // Change load of spilled value to only a spill
16535 // instruct storeI(memory mem, iRegI src)
16536 // %{
16537 // match(Set mem (StoreI mem src));
16538 // %}
16539 //
16540 // instruct loadI(iRegINoSp dst, memory mem)
16541 // %{
16542 // match(Set dst (LoadI mem));
16543 // %}
16544 //
16545
16546 //----------SMARTSPILL RULES---------------------------------------------------
16547 // These must follow all instruction definitions as they use the names
16548 // defined in the instructions definitions.
16549
16550 // Local Variables:
16551 // mode: c++
16552 // End: