1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else {
1688 return 6 * NativeInstruction::instruction_size;
1689 }
1690 }
1691
1692 //=============================================================================
1693
1694 #ifndef PRODUCT
1695 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1696 st->print("BREAKPOINT");
1697 }
1698 #endif
1699
1700 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1701 __ brk(0);
1702 }
1703
1704 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1705 return MachNode::size(ra_);
1706 }
1707
1708 //=============================================================================
1709
1710 #ifndef PRODUCT
1711 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1712 st->print("nop \t# %d bytes pad for loops and calls", _count);
1713 }
1714 #endif
1715
1716 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1717 for (int i = 0; i < _count; i++) {
1718 __ nop();
1719 }
1720 }
1721
1722 uint MachNopNode::size(PhaseRegAlloc*) const {
1723 return _count * NativeInstruction::instruction_size;
1724 }
1725
1726 //=============================================================================
1727 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1728
1729 int ConstantTable::calculate_table_base_offset() const {
1730 return 0; // absolute addressing, no offset
1731 }
1732
1733 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1734 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1735 ShouldNotReachHere();
1736 }
1737
1738 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1739 // Empty encoding
1740 }
1741
1742 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1743 return 0;
1744 }
1745
1746 #ifndef PRODUCT
1747 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1748 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1749 }
1750 #endif
1751
1752 #ifndef PRODUCT
1753 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1754 Compile* C = ra_->C;
1755
1756 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1757
1758 if (C->output()->need_stack_bang(framesize))
1759 st->print("# stack bang size=%d\n\t", framesize);
1760
1761 if (VM_Version::use_rop_protection()) {
1762 st->print("ldr zr, [lr]\n\t");
1763 st->print("paciaz\n\t");
1764 }
1765 if (framesize < ((1 << 9) + 2 * wordSize)) {
1766 st->print("sub sp, sp, #%d\n\t", framesize);
1767 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1768 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1769 } else {
1770 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1771 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1772 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1773 st->print("sub sp, sp, rscratch1");
1774 }
1775 if (C->stub_function() == nullptr) {
1776 st->print("\n\t");
1777 st->print("ldr rscratch1, [guard]\n\t");
1778 st->print("dmb ishld\n\t");
1779 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1780 st->print("cmp rscratch1, rscratch2\n\t");
1781 st->print("b.eq skip");
1782 st->print("\n\t");
1783 st->print("blr #nmethod_entry_barrier_stub\n\t");
1784 st->print("b skip\n\t");
1785 st->print("guard: int\n\t");
1786 st->print("\n\t");
1787 st->print("skip:\n\t");
1788 }
1789 }
1790 #endif
1791
1792 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1793 Compile* C = ra_->C;
1794
1795 // n.b. frame size includes space for return pc and rfp
1796 const int framesize = C->output()->frame_size_in_bytes();
1797
1798 if (C->clinit_barrier_on_entry()) {
1799 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1800
1801 Label L_skip_barrier;
1802
1803 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1804 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1805 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1806 __ bind(L_skip_barrier);
1807 }
1808
1809 if (C->max_vector_size() > 0) {
1810 __ reinitialize_ptrue();
1811 }
1812
1813 int bangsize = C->output()->bang_size_in_bytes();
1814 if (C->output()->need_stack_bang(bangsize))
1815 __ generate_stack_overflow_check(bangsize);
1816
1817 __ build_frame(framesize);
1818
1819 if (C->stub_function() == nullptr) {
1820 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1821 // Dummy labels for just measuring the code size
1822 Label dummy_slow_path;
1823 Label dummy_continuation;
1824 Label dummy_guard;
1825 Label* slow_path = &dummy_slow_path;
1826 Label* continuation = &dummy_continuation;
1827 Label* guard = &dummy_guard;
1828 if (!Compile::current()->output()->in_scratch_emit_size()) {
1829 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1830 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1831 Compile::current()->output()->add_stub(stub);
1832 slow_path = &stub->entry();
1833 continuation = &stub->continuation();
1834 guard = &stub->guard();
1835 }
1836 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1837 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1838 }
1839
1840 if (VerifyStackAtCalls) {
1841 Unimplemented();
1842 }
1843
1844 C->output()->set_frame_complete(__ offset());
1845
1846 if (C->has_mach_constant_base_node()) {
1847 // NOTE: We set the table base offset here because users might be
1848 // emitted before MachConstantBaseNode.
1849 ConstantTable& constant_table = C->output()->constant_table();
1850 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1851 }
1852 }
1853
1854 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1855 {
1856 return MachNode::size(ra_); // too many variables; just compute it
1857 // the hard way
1858 }
1859
1860 int MachPrologNode::reloc() const
1861 {
1862 return 0;
1863 }
1864
1865 //=============================================================================
1866
1867 #ifndef PRODUCT
1868 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1869 Compile* C = ra_->C;
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 st->print("# pop frame %d\n\t",framesize);
1873
1874 if (framesize == 0) {
1875 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1876 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1877 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1878 st->print("add sp, sp, #%d\n\t", framesize);
1879 } else {
1880 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1881 st->print("add sp, sp, rscratch1\n\t");
1882 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1883 }
1884 if (VM_Version::use_rop_protection()) {
1885 st->print("autiaz\n\t");
1886 st->print("ldr zr, [lr]\n\t");
1887 }
1888
1889 if (do_polling() && C->is_method_compilation()) {
1890 st->print("# test polling word\n\t");
1891 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1892 st->print("cmp sp, rscratch1\n\t");
1893 st->print("bhi #slow_path");
1894 }
1895 }
1896 #endif
1897
1898 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1899 Compile* C = ra_->C;
1900 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1901
1902 __ remove_frame(framesize);
1903
1904 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1905 __ reserved_stack_check();
1906 }
1907
1908 if (do_polling() && C->is_method_compilation()) {
1909 Label dummy_label;
1910 Label* code_stub = &dummy_label;
1911 if (!C->output()->in_scratch_emit_size()) {
1912 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1913 C->output()->add_stub(stub);
1914 code_stub = &stub->entry();
1915 }
1916 __ relocate(relocInfo::poll_return_type);
1917 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1918 }
1919 }
1920
1921 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1922 // Variable size. Determine dynamically.
1923 return MachNode::size(ra_);
1924 }
1925
1926 int MachEpilogNode::reloc() const {
1927 // Return number of relocatable values contained in this instruction.
1928 return 1; // 1 for polling page.
1929 }
1930
1931 const Pipeline * MachEpilogNode::pipeline() const {
1932 return MachNode::pipeline_class();
1933 }
1934
1935 //=============================================================================
1936
1937 static enum RC rc_class(OptoReg::Name reg) {
1938
1939 if (reg == OptoReg::Bad) {
1940 return rc_bad;
1941 }
1942
1943 // we have 32 int registers * 2 halves
1944 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1945
1946 if (reg < slots_of_int_registers) {
1947 return rc_int;
1948 }
1949
1950 // we have 32 float register * 8 halves
1951 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1952 if (reg < slots_of_int_registers + slots_of_float_registers) {
1953 return rc_float;
1954 }
1955
1956 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1957 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1958 return rc_predicate;
1959 }
1960
1961 // Between predicate regs & stack is the flags.
1962 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1963
1964 return rc_stack;
1965 }
1966
1967 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1968 Compile* C = ra_->C;
1969
1970 // Get registers to move.
1971 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1972 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1973 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1974 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1975
1976 enum RC src_hi_rc = rc_class(src_hi);
1977 enum RC src_lo_rc = rc_class(src_lo);
1978 enum RC dst_hi_rc = rc_class(dst_hi);
1979 enum RC dst_lo_rc = rc_class(dst_lo);
1980
1981 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1982
1983 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1984 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1985 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1986 "expected aligned-adjacent pairs");
1987 }
1988
1989 if (src_lo == dst_lo && src_hi == dst_hi) {
1990 return 0; // Self copy, no move.
1991 }
1992
1993 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1994 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1995 int src_offset = ra_->reg2offset(src_lo);
1996 int dst_offset = ra_->reg2offset(dst_lo);
1997
1998 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
1999 uint ireg = ideal_reg();
2000 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2001 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2002 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2003 if (ireg == Op_VecA && masm) {
2004 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2005 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2006 // stack->stack
2007 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2008 sve_vector_reg_size_in_bytes);
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2013 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2016 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]),
2018 as_FloatRegister(Matcher::_regEncode[src_lo]));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 } else if (masm) {
2023 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2024 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2025 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2026 // stack->stack
2027 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2028 if (ireg == Op_VecD) {
2029 __ unspill(rscratch1, true, src_offset);
2030 __ spill(rscratch1, true, dst_offset);
2031 } else {
2032 __ spill_copy128(src_offset, dst_offset);
2033 }
2034 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2035 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2036 ireg == Op_VecD ? __ T8B : __ T16B,
2037 as_FloatRegister(Matcher::_regEncode[src_lo]));
2038 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2039 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2040 ireg == Op_VecD ? __ D : __ Q,
2041 ra_->reg2offset(dst_lo));
2042 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2043 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 ireg == Op_VecD ? __ D : __ Q,
2045 ra_->reg2offset(src_lo));
2046 } else {
2047 ShouldNotReachHere();
2048 }
2049 }
2050 } else if (masm) {
2051 switch (src_lo_rc) {
2052 case rc_int:
2053 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2054 if (is64) {
2055 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_Register(Matcher::_regEncode[src_lo]));
2057 } else {
2058 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 }
2061 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2062 if (is64) {
2063 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_Register(Matcher::_regEncode[src_lo]));
2065 } else {
2066 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 }
2069 } else { // gpr --> stack spill
2070 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2071 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2072 }
2073 break;
2074 case rc_float:
2075 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2076 if (is64) {
2077 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2078 as_FloatRegister(Matcher::_regEncode[src_lo]));
2079 } else {
2080 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 }
2083 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2084 if (is64) {
2085 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2086 as_FloatRegister(Matcher::_regEncode[src_lo]));
2087 } else {
2088 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 }
2091 } else { // fpr --> stack spill
2092 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2093 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2094 is64 ? __ D : __ S, dst_offset);
2095 }
2096 break;
2097 case rc_stack:
2098 if (dst_lo_rc == rc_int) { // stack --> gpr load
2099 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2100 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2101 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2102 is64 ? __ D : __ S, src_offset);
2103 } else if (dst_lo_rc == rc_predicate) {
2104 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2105 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2106 } else { // stack --> stack copy
2107 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2108 if (ideal_reg() == Op_RegVectMask) {
2109 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2110 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2111 } else {
2112 __ unspill(rscratch1, is64, src_offset);
2113 __ spill(rscratch1, is64, dst_offset);
2114 }
2115 }
2116 break;
2117 case rc_predicate:
2118 if (dst_lo_rc == rc_predicate) {
2119 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2120 } else if (dst_lo_rc == rc_stack) {
2121 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2122 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2123 } else {
2124 assert(false, "bad src and dst rc_class combination.");
2125 ShouldNotReachHere();
2126 }
2127 break;
2128 default:
2129 assert(false, "bad rc_class for spill");
2130 ShouldNotReachHere();
2131 }
2132 }
2133
2134 if (st) {
2135 st->print("spill ");
2136 if (src_lo_rc == rc_stack) {
2137 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2138 } else {
2139 st->print("%s -> ", Matcher::regName[src_lo]);
2140 }
2141 if (dst_lo_rc == rc_stack) {
2142 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2143 } else {
2144 st->print("%s", Matcher::regName[dst_lo]);
2145 }
2146 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2147 int vsize = 0;
2148 switch (ideal_reg()) {
2149 case Op_VecD:
2150 vsize = 64;
2151 break;
2152 case Op_VecX:
2153 vsize = 128;
2154 break;
2155 case Op_VecA:
2156 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2157 break;
2158 default:
2159 assert(false, "bad register type for spill");
2160 ShouldNotReachHere();
2161 }
2162 st->print("\t# vector spill size = %d", vsize);
2163 } else if (ideal_reg() == Op_RegVectMask) {
2164 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2165 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2166 st->print("\t# predicate spill size = %d", vsize);
2167 } else {
2168 st->print("\t# spill size = %d", is64 ? 64 : 32);
2169 }
2170 }
2171
2172 return 0;
2173
2174 }
2175
2176 #ifndef PRODUCT
2177 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2178 if (!ra_)
2179 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2180 else
2181 implementation(nullptr, ra_, false, st);
2182 }
2183 #endif
2184
2185 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2186 implementation(masm, ra_, false, nullptr);
2187 }
2188
2189 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2190 return MachNode::size(ra_);
2191 }
2192
2193 //=============================================================================
2194
2195 #ifndef PRODUCT
2196 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2197 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2198 int reg = ra_->get_reg_first(this);
2199 st->print("add %s, rsp, #%d]\t# box lock",
2200 Matcher::regName[reg], offset);
2201 }
2202 #endif
2203
2204 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2205 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2206 int reg = ra_->get_encode(this);
2207
2208 // This add will handle any 24-bit signed offset. 24 bits allows an
2209 // 8 megabyte stack frame.
2210 __ add(as_Register(reg), sp, offset);
2211 }
2212
2213 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2214 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2215 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2216
2217 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2218 return NativeInstruction::instruction_size;
2219 } else {
2220 return 2 * NativeInstruction::instruction_size;
2221 }
2222 }
2223
2224 //=============================================================================
2225
2226 #ifndef PRODUCT
2227 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2228 {
2229 st->print_cr("# MachUEPNode");
2230 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2231 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2232 st->print_cr("\tcmpw rscratch1, r10");
2233 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2234 }
2235 #endif
2236
2237 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2238 {
2239 __ ic_check(InteriorEntryAlignment);
2240 }
2241
2242 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2243 {
2244 return MachNode::size(ra_);
2245 }
2246
2247 // REQUIRED EMIT CODE
2248
2249 //=============================================================================
2250
2251 // Emit deopt handler code.
2252 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2253 {
2254 // Note that the code buffer's insts_mark is always relative to insts.
2255 // That's why we must use the macroassembler to generate a handler.
2256 address base = __ start_a_stub(size_deopt_handler());
2257 if (base == nullptr) {
2258 ciEnv::current()->record_failure("CodeCache is full");
2259 return 0; // CodeBuffer::expand failed
2260 }
2261
2262 int offset = __ offset();
2263 Label start;
2264 __ bind(start);
2265 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2266
2267 int entry_offset = __ offset();
2268 __ b(start);
2269
2270 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2271 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2272 "out of bounds read in post-call NOP check");
2273 __ end_a_stub();
2274 return entry_offset;
2275 }
2276
2277 // REQUIRED MATCHER CODE
2278
2279 //=============================================================================
2280
2281 bool Matcher::match_rule_supported(int opcode) {
2282 if (!has_match_rule(opcode))
2283 return false;
2284
2285 switch (opcode) {
2286 case Op_OnSpinWait:
2287 return VM_Version::supports_on_spin_wait();
2288 case Op_CacheWB:
2289 case Op_CacheWBPreSync:
2290 case Op_CacheWBPostSync:
2291 if (!VM_Version::supports_data_cache_line_flush()) {
2292 return false;
2293 }
2294 break;
2295 case Op_ExpandBits:
2296 case Op_CompressBits:
2297 if (!VM_Version::supports_svebitperm()) {
2298 return false;
2299 }
2300 break;
2301 case Op_FmaF:
2302 case Op_FmaD:
2303 case Op_FmaVF:
2304 case Op_FmaVD:
2305 if (!UseFMA) {
2306 return false;
2307 }
2308 break;
2309 case Op_FmaHF:
2310 // UseFMA flag also needs to be checked along with FEAT_FP16
2311 if (!UseFMA || !is_feat_fp16_supported()) {
2312 return false;
2313 }
2314 break;
2315 case Op_AddHF:
2316 case Op_SubHF:
2317 case Op_MulHF:
2318 case Op_DivHF:
2319 case Op_MinHF:
2320 case Op_MaxHF:
2321 case Op_SqrtHF:
2322 // Half-precision floating point scalar operations require FEAT_FP16
2323 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2324 // features are supported.
2325 if (!is_feat_fp16_supported()) {
2326 return false;
2327 }
2328 break;
2329 }
2330
2331 return true; // Per default match rules are supported.
2332 }
2333
2334 const RegMask* Matcher::predicate_reg_mask(void) {
2335 return &_PR_REG_mask;
2336 }
2337
2338 bool Matcher::supports_vector_calling_convention(void) {
2339 return EnableVectorSupport;
2340 }
2341
2342 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2343 assert(EnableVectorSupport, "sanity");
2344 int lo = V0_num;
2345 int hi = V0_H_num;
2346 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2347 hi = V0_K_num;
2348 }
2349 return OptoRegPair(hi, lo);
2350 }
2351
2352 // Is this branch offset short enough that a short branch can be used?
2353 //
2354 // NOTE: If the platform does not provide any short branch variants, then
2355 // this method should return false for offset 0.
2356 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2357 // The passed offset is relative to address of the branch.
2358
2359 return (-32768 <= offset && offset < 32768);
2360 }
2361
2362 // Vector width in bytes.
2363 int Matcher::vector_width_in_bytes(BasicType bt) {
2364 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2365 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2366 // Minimum 2 values in vector
2367 if (size < 2*type2aelembytes(bt)) size = 0;
2368 // But never < 4
2369 if (size < 4) size = 0;
2370 return size;
2371 }
2372
2373 // Limits on vector size (number of elements) loaded into vector.
2374 int Matcher::max_vector_size(const BasicType bt) {
2375 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2376 }
2377
2378 int Matcher::min_vector_size(const BasicType bt) {
2379 // Usually, the shortest vector length supported by AArch64 ISA and
2380 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2381 // vectors in a few special cases.
2382 int size;
2383 switch(bt) {
2384 case T_BOOLEAN:
2385 // Load/store a vector mask with only 2 elements for vector types
2386 // such as "2I/2F/2L/2D".
2387 size = 2;
2388 break;
2389 case T_BYTE:
2390 // Generate a "4B" vector, to support vector cast between "8B/16B"
2391 // and "4S/4I/4L/4F/4D".
2392 size = 4;
2393 break;
2394 case T_SHORT:
2395 // Generate a "2S" vector, to support vector cast between "4S/8S"
2396 // and "2I/2L/2F/2D".
2397 size = 2;
2398 break;
2399 default:
2400 // Limit the min vector length to 64-bit.
2401 size = 8 / type2aelembytes(bt);
2402 // The number of elements in a vector should be at least 2.
2403 size = MAX2(size, 2);
2404 }
2405
2406 int max_size = max_vector_size(bt);
2407 return MIN2(size, max_size);
2408 }
2409
2410 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2411 return Matcher::max_vector_size(bt);
2412 }
2413
2414 // Actual max scalable vector register length.
2415 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2416 return Matcher::max_vector_size(bt);
2417 }
2418
2419 // Vector ideal reg.
2420 uint Matcher::vector_ideal_reg(int len) {
2421 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2422 return Op_VecA;
2423 }
2424 switch(len) {
2425 // For 16-bit/32-bit mask vector, reuse VecD.
2426 case 2:
2427 case 4:
2428 case 8: return Op_VecD;
2429 case 16: return Op_VecX;
2430 }
2431 ShouldNotReachHere();
2432 return 0;
2433 }
2434
2435 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2436 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2437 switch (ideal_reg) {
2438 case Op_VecA: return new vecAOper();
2439 case Op_VecD: return new vecDOper();
2440 case Op_VecX: return new vecXOper();
2441 }
2442 ShouldNotReachHere();
2443 return nullptr;
2444 }
2445
2446 bool Matcher::is_reg2reg_move(MachNode* m) {
2447 return false;
2448 }
2449
2450 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2451 return false;
2452 }
2453
2454 bool Matcher::is_generic_vector(MachOper* opnd) {
2455 return opnd->opcode() == VREG;
2456 }
2457
2458 #ifdef ASSERT
2459 // Return whether or not this register is ever used as an argument.
2460 bool Matcher::can_be_java_arg(int reg)
2461 {
2462 return
2463 reg == R0_num || reg == R0_H_num ||
2464 reg == R1_num || reg == R1_H_num ||
2465 reg == R2_num || reg == R2_H_num ||
2466 reg == R3_num || reg == R3_H_num ||
2467 reg == R4_num || reg == R4_H_num ||
2468 reg == R5_num || reg == R5_H_num ||
2469 reg == R6_num || reg == R6_H_num ||
2470 reg == R7_num || reg == R7_H_num ||
2471 reg == V0_num || reg == V0_H_num ||
2472 reg == V1_num || reg == V1_H_num ||
2473 reg == V2_num || reg == V2_H_num ||
2474 reg == V3_num || reg == V3_H_num ||
2475 reg == V4_num || reg == V4_H_num ||
2476 reg == V5_num || reg == V5_H_num ||
2477 reg == V6_num || reg == V6_H_num ||
2478 reg == V7_num || reg == V7_H_num;
2479 }
2480 #endif
2481
2482 uint Matcher::int_pressure_limit()
2483 {
2484 // JDK-8183543: When taking the number of available registers as int
2485 // register pressure threshold, the jtreg test:
2486 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2487 // failed due to C2 compilation failure with
2488 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2489 //
2490 // A derived pointer is live at CallNode and then is flagged by RA
2491 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2492 // derived pointers and lastly fail to spill after reaching maximum
2493 // number of iterations. Lowering the default pressure threshold to
2494 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2495 // a high register pressure area of the code so that split_DEF can
2496 // generate DefinitionSpillCopy for the derived pointer.
2497 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2498 if (!PreserveFramePointer) {
2499 // When PreserveFramePointer is off, frame pointer is allocatable,
2500 // but different from other SOC registers, it is excluded from
2501 // fatproj's mask because its save type is No-Save. Decrease 1 to
2502 // ensure high pressure at fatproj when PreserveFramePointer is off.
2503 // See check_pressure_at_fatproj().
2504 default_int_pressure_threshold--;
2505 }
2506 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2507 }
2508
2509 uint Matcher::float_pressure_limit()
2510 {
2511 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2512 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2513 }
2514
2515 const RegMask& Matcher::divI_proj_mask() {
2516 ShouldNotReachHere();
2517 return RegMask::EMPTY;
2518 }
2519
2520 // Register for MODI projection of divmodI.
2521 const RegMask& Matcher::modI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for DIVL projection of divmodL.
2527 const RegMask& Matcher::divL_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for MODL projection of divmodL.
2533 const RegMask& Matcher::modL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2539 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2540 Node* u = addp->fast_out(i);
2541 if (u->is_LoadStore()) {
2542 // On AArch64, LoadStoreNodes (i.e. compare and swap
2543 // instructions) only take register indirect as an operand, so
2544 // any attempt to use an AddPNode as an input to a LoadStoreNode
2545 // must fail.
2546 return false;
2547 }
2548 if (u->is_Mem()) {
2549 int opsize = u->as_Mem()->memory_size();
2550 assert(opsize > 0, "unexpected memory operand size");
2551 if (u->as_Mem()->memory_size() != (1<<shift)) {
2552 return false;
2553 }
2554 }
2555 }
2556 return true;
2557 }
2558
2559 // Convert BoolTest condition to Assembler condition.
2560 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2561 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2562 Assembler::Condition result;
2563 switch(cond) {
2564 case BoolTest::eq:
2565 result = Assembler::EQ; break;
2566 case BoolTest::ne:
2567 result = Assembler::NE; break;
2568 case BoolTest::le:
2569 result = Assembler::LE; break;
2570 case BoolTest::ge:
2571 result = Assembler::GE; break;
2572 case BoolTest::lt:
2573 result = Assembler::LT; break;
2574 case BoolTest::gt:
2575 result = Assembler::GT; break;
2576 case BoolTest::ule:
2577 result = Assembler::LS; break;
2578 case BoolTest::uge:
2579 result = Assembler::HS; break;
2580 case BoolTest::ult:
2581 result = Assembler::LO; break;
2582 case BoolTest::ugt:
2583 result = Assembler::HI; break;
2584 case BoolTest::overflow:
2585 result = Assembler::VS; break;
2586 case BoolTest::no_overflow:
2587 result = Assembler::VC; break;
2588 default:
2589 ShouldNotReachHere();
2590 return Assembler::Condition(-1);
2591 }
2592
2593 // Check conversion
2594 if (cond & BoolTest::unsigned_compare) {
2595 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2596 } else {
2597 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2598 }
2599
2600 return result;
2601 }
2602
2603 // Binary src (Replicate con)
2604 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2605 if (n == nullptr || m == nullptr) {
2606 return false;
2607 }
2608
2609 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2610 return false;
2611 }
2612
2613 Node* imm_node = m->in(1);
2614 if (!imm_node->is_Con()) {
2615 return false;
2616 }
2617
2618 const Type* t = imm_node->bottom_type();
2619 if (!(t->isa_int() || t->isa_long())) {
2620 return false;
2621 }
2622
2623 switch (n->Opcode()) {
2624 case Op_AndV:
2625 case Op_OrV:
2626 case Op_XorV: {
2627 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2628 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2629 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2630 }
2631 case Op_AddVB:
2632 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2633 case Op_AddVS:
2634 case Op_AddVI:
2635 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2636 case Op_AddVL:
2637 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2638 default:
2639 return false;
2640 }
2641 }
2642
2643 // (XorV src (Replicate m1))
2644 // (XorVMask src (MaskAll m1))
2645 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2646 if (n != nullptr && m != nullptr) {
2647 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2648 VectorNode::is_all_ones_vector(m);
2649 }
2650 return false;
2651 }
2652
2653 // Should the matcher clone input 'm' of node 'n'?
2654 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2655 if (is_vshift_con_pattern(n, m) ||
2656 is_vector_bitwise_not_pattern(n, m) ||
2657 is_valid_sve_arith_imm_pattern(n, m) ||
2658 is_encode_and_store_pattern(n, m)) {
2659 mstack.push(m, Visit);
2660 return true;
2661 }
2662 return false;
2663 }
2664
2665 // Should the Matcher clone shifts on addressing modes, expecting them
2666 // to be subsumed into complex addressing expressions or compute them
2667 // into registers?
2668 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2669
2670 // Loads and stores with indirect memory input (e.g., volatile loads and
2671 // stores) do not subsume the input into complex addressing expressions. If
2672 // the addressing expression is input to at least one such load or store, do
2673 // not clone the addressing expression. Query needs_acquiring_load and
2674 // needs_releasing_store as a proxy for indirect memory input, as it is not
2675 // possible to directly query for indirect memory input at this stage.
2676 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2677 Node* n = m->fast_out(i);
2678 if (n->is_Load() && needs_acquiring_load(n)) {
2679 return false;
2680 }
2681 if (n->is_Store() && needs_releasing_store(n)) {
2682 return false;
2683 }
2684 }
2685
2686 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2687 return true;
2688 }
2689
2690 Node *off = m->in(AddPNode::Offset);
2691 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2692 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2693 // Are there other uses besides address expressions?
2694 !is_visited(off)) {
2695 address_visited.set(off->_idx); // Flag as address_visited
2696 mstack.push(off->in(2), Visit);
2697 Node *conv = off->in(1);
2698 if (conv->Opcode() == Op_ConvI2L &&
2699 // Are there other uses besides address expressions?
2700 !is_visited(conv)) {
2701 address_visited.set(conv->_idx); // Flag as address_visited
2702 mstack.push(conv->in(1), Pre_Visit);
2703 } else {
2704 mstack.push(conv, Pre_Visit);
2705 }
2706 address_visited.test_set(m->_idx); // Flag as address_visited
2707 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2708 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2709 return true;
2710 } else if (off->Opcode() == Op_ConvI2L &&
2711 // Are there other uses besides address expressions?
2712 !is_visited(off)) {
2713 address_visited.test_set(m->_idx); // Flag as address_visited
2714 address_visited.set(off->_idx); // Flag as address_visited
2715 mstack.push(off->in(1), Pre_Visit);
2716 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2717 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2718 return true;
2719 }
2720 return false;
2721 }
2722
2723 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2724 { \
2725 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2726 guarantee(DISP == 0, "mode not permitted for volatile"); \
2727 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2728 __ INSN(REG, as_Register(BASE)); \
2729 }
2730
2731
2732 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2733 {
2734 Address::extend scale;
2735
2736 // Hooboy, this is fugly. We need a way to communicate to the
2737 // encoder that the index needs to be sign extended, so we have to
2738 // enumerate all the cases.
2739 switch (opcode) {
2740 case INDINDEXSCALEDI2L:
2741 case INDINDEXSCALEDI2LN:
2742 case INDINDEXI2L:
2743 case INDINDEXI2LN:
2744 scale = Address::sxtw(size);
2745 break;
2746 default:
2747 scale = Address::lsl(size);
2748 }
2749
2750 if (index == -1) {
2751 return Address(base, disp);
2752 } else {
2753 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2754 return Address(base, as_Register(index), scale);
2755 }
2756 }
2757
2758
2759 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2760 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2761 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2762 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2763 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2764
2765 // Used for all non-volatile memory accesses. The use of
2766 // $mem->opcode() to discover whether this pattern uses sign-extended
2767 // offsets is something of a kludge.
2768 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2769 Register reg, int opcode,
2770 Register base, int index, int scale, int disp,
2771 int size_in_memory)
2772 {
2773 Address addr = mem2address(opcode, base, index, scale, disp);
2774 if (addr.getMode() == Address::base_plus_offset) {
2775 /* Fix up any out-of-range offsets. */
2776 assert_different_registers(rscratch1, base);
2777 assert_different_registers(rscratch1, reg);
2778 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2779 }
2780 (masm->*insn)(reg, addr);
2781 }
2782
2783 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2784 FloatRegister reg, int opcode,
2785 Register base, int index, int size, int disp,
2786 int size_in_memory)
2787 {
2788 Address::extend scale;
2789
2790 switch (opcode) {
2791 case INDINDEXSCALEDI2L:
2792 case INDINDEXSCALEDI2LN:
2793 scale = Address::sxtw(size);
2794 break;
2795 default:
2796 scale = Address::lsl(size);
2797 }
2798
2799 if (index == -1) {
2800 // Fix up any out-of-range offsets.
2801 assert_different_registers(rscratch1, base);
2802 Address addr = Address(base, disp);
2803 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2804 (masm->*insn)(reg, addr);
2805 } else {
2806 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2807 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2808 }
2809 }
2810
2811 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2812 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2813 int opcode, Register base, int index, int size, int disp)
2814 {
2815 if (index == -1) {
2816 (masm->*insn)(reg, T, Address(base, disp));
2817 } else {
2818 assert(disp == 0, "unsupported address mode");
2819 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2820 }
2821 }
2822
2823 %}
2824
2825
2826
2827 //----------ENCODING BLOCK-----------------------------------------------------
2828 // This block specifies the encoding classes used by the compiler to
2829 // output byte streams. Encoding classes are parameterized macros
2830 // used by Machine Instruction Nodes in order to generate the bit
2831 // encoding of the instruction. Operands specify their base encoding
2832 // interface with the interface keyword. There are currently
2833 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2834 // COND_INTER. REG_INTER causes an operand to generate a function
2835 // which returns its register number when queried. CONST_INTER causes
2836 // an operand to generate a function which returns the value of the
2837 // constant when queried. MEMORY_INTER causes an operand to generate
2838 // four functions which return the Base Register, the Index Register,
2839 // the Scale Value, and the Offset Value of the operand when queried.
2840 // COND_INTER causes an operand to generate six functions which return
2841 // the encoding code (ie - encoding bits for the instruction)
2842 // associated with each basic boolean condition for a conditional
2843 // instruction.
2844 //
2845 // Instructions specify two basic values for encoding. Again, a
2846 // function is available to check if the constant displacement is an
2847 // oop. They use the ins_encode keyword to specify their encoding
2848 // classes (which must be a sequence of enc_class names, and their
2849 // parameters, specified in the encoding block), and they use the
2850 // opcode keyword to specify, in order, their primary, secondary, and
2851 // tertiary opcode. Only the opcode sections which a particular
2852 // instruction needs for encoding need to be specified.
2853 encode %{
2854 // Build emit functions for each basic byte or larger field in the
2855 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2856 // from C++ code in the enc_class source block. Emit functions will
2857 // live in the main source block for now. In future, we can
2858 // generalize this by adding a syntax that specifies the sizes of
2859 // fields in an order, so that the adlc can build the emit functions
2860 // automagically
2861
2862 // catch all for unimplemented encodings
2863 enc_class enc_unimplemented %{
2864 __ unimplemented("C2 catch all");
2865 %}
2866
2867 // BEGIN Non-volatile memory access
2868
2869 // This encoding class is generated automatically from ad_encode.m4.
2870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2871 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2872 Register dst_reg = as_Register($dst$$reg);
2873 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2874 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2875 %}
2876
2877 // This encoding class is generated automatically from ad_encode.m4.
2878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2879 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2880 Register dst_reg = as_Register($dst$$reg);
2881 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2883 %}
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2968 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2976 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2984 Register src_reg = as_Register($src$$reg);
2985 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_strb0(memory1 mem) %{
2992 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2993 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2994 %}
2995
2996 // This encoding class is generated automatically from ad_encode.m4.
2997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2998 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2999 Register src_reg = as_Register($src$$reg);
3000 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3001 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3002 %}
3003
3004 // This encoding class is generated automatically from ad_encode.m4.
3005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3006 enc_class aarch64_enc_strh0(memory2 mem) %{
3007 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3008 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3009 %}
3010
3011 // This encoding class is generated automatically from ad_encode.m4.
3012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3013 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3014 Register src_reg = as_Register($src$$reg);
3015 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strw0(memory4 mem) %{
3022 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3029 Register src_reg = as_Register($src$$reg);
3030 // we sometimes get asked to store the stack pointer into the
3031 // current thread -- we cannot do that directly on AArch64
3032 if (src_reg == r31_sp) {
3033 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3034 __ mov(rscratch2, sp);
3035 src_reg = rscratch2;
3036 }
3037 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_str0(memory8 mem) %{
3044 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3051 FloatRegister src_reg = as_FloatRegister($src$$reg);
3052 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3054 %}
3055
3056 // This encoding class is generated automatically from ad_encode.m4.
3057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3058 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3059 FloatRegister src_reg = as_FloatRegister($src$$reg);
3060 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3067 __ membar(Assembler::StoreStore);
3068 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3069 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3070 %}
3071
3072 // END Non-volatile memory access
3073
3074 // Vector loads and stores
3075 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3076 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3077 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3078 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3079 %}
3080
3081 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3082 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3083 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3084 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3085 %}
3086
3087 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3088 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3089 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3090 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3091 %}
3092
3093 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3100 FloatRegister src_reg = as_FloatRegister($src$$reg);
3101 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3106 FloatRegister src_reg = as_FloatRegister($src$$reg);
3107 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3112 FloatRegister src_reg = as_FloatRegister($src$$reg);
3113 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 // volatile loads and stores
3124
3125 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3126 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3127 rscratch1, stlrb);
3128 %}
3129
3130 enc_class aarch64_enc_stlrb0(memory mem) %{
3131 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3132 rscratch1, stlrb);
3133 %}
3134
3135 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3136 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, stlrh);
3138 %}
3139
3140 enc_class aarch64_enc_stlrh0(memory mem) %{
3141 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, stlrh);
3143 %}
3144
3145 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3146 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, stlrw);
3148 %}
3149
3150 enc_class aarch64_enc_stlrw0(memory mem) %{
3151 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3152 rscratch1, stlrw);
3153 %}
3154
3155 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3156 Register dst_reg = as_Register($dst$$reg);
3157 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, ldarb);
3159 __ sxtbw(dst_reg, dst_reg);
3160 %}
3161
3162 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3163 Register dst_reg = as_Register($dst$$reg);
3164 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, ldarb);
3166 __ sxtb(dst_reg, dst_reg);
3167 %}
3168
3169 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarb);
3172 %}
3173
3174 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarb);
3177 %}
3178
3179 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3180 Register dst_reg = as_Register($dst$$reg);
3181 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarh);
3183 __ sxthw(dst_reg, dst_reg);
3184 %}
3185
3186 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3187 Register dst_reg = as_Register($dst$$reg);
3188 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarh);
3190 __ sxth(dst_reg, dst_reg);
3191 %}
3192
3193 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3194 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarh);
3196 %}
3197
3198 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3199 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarh);
3201 %}
3202
3203 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3204 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarw);
3206 %}
3207
3208 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3209 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldarw);
3211 %}
3212
3213 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3214 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3215 rscratch1, ldar);
3216 %}
3217
3218 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3219 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3220 rscratch1, ldarw);
3221 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3222 %}
3223
3224 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3225 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldar);
3227 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3228 %}
3229
3230 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3231 Register src_reg = as_Register($src$$reg);
3232 // we sometimes get asked to store the stack pointer into the
3233 // current thread -- we cannot do that directly on AArch64
3234 if (src_reg == r31_sp) {
3235 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3236 __ mov(rscratch2, sp);
3237 src_reg = rscratch2;
3238 }
3239 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3240 rscratch1, stlr);
3241 %}
3242
3243 enc_class aarch64_enc_stlr0(memory mem) %{
3244 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3245 rscratch1, stlr);
3246 %}
3247
3248 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3249 {
3250 FloatRegister src_reg = as_FloatRegister($src$$reg);
3251 __ fmovs(rscratch2, src_reg);
3252 }
3253 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3254 rscratch1, stlrw);
3255 %}
3256
3257 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3258 {
3259 FloatRegister src_reg = as_FloatRegister($src$$reg);
3260 __ fmovd(rscratch2, src_reg);
3261 }
3262 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, stlr);
3264 %}
3265
3266 // synchronized read/update encodings
3267
3268 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3269 Register dst_reg = as_Register($dst$$reg);
3270 Register base = as_Register($mem$$base);
3271 int index = $mem$$index;
3272 int scale = $mem$$scale;
3273 int disp = $mem$$disp;
3274 if (index == -1) {
3275 if (disp != 0) {
3276 __ lea(rscratch1, Address(base, disp));
3277 __ ldaxr(dst_reg, rscratch1);
3278 } else {
3279 // TODO
3280 // should we ever get anything other than this case?
3281 __ ldaxr(dst_reg, base);
3282 }
3283 } else {
3284 Register index_reg = as_Register(index);
3285 if (disp == 0) {
3286 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3287 __ ldaxr(dst_reg, rscratch1);
3288 } else {
3289 __ lea(rscratch1, Address(base, disp));
3290 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3291 __ ldaxr(dst_reg, rscratch1);
3292 }
3293 }
3294 %}
3295
3296 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3297 Register src_reg = as_Register($src$$reg);
3298 Register base = as_Register($mem$$base);
3299 int index = $mem$$index;
3300 int scale = $mem$$scale;
3301 int disp = $mem$$disp;
3302 if (index == -1) {
3303 if (disp != 0) {
3304 __ lea(rscratch2, Address(base, disp));
3305 __ stlxr(rscratch1, src_reg, rscratch2);
3306 } else {
3307 // TODO
3308 // should we ever get anything other than this case?
3309 __ stlxr(rscratch1, src_reg, base);
3310 }
3311 } else {
3312 Register index_reg = as_Register(index);
3313 if (disp == 0) {
3314 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3315 __ stlxr(rscratch1, src_reg, rscratch2);
3316 } else {
3317 __ lea(rscratch2, Address(base, disp));
3318 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3319 __ stlxr(rscratch1, src_reg, rscratch2);
3320 }
3321 }
3322 __ cmpw(rscratch1, zr);
3323 %}
3324
3325 // prefetch encodings
3326
3327 enc_class aarch64_enc_prefetchw(memory mem) %{
3328 Register base = as_Register($mem$$base);
3329 int index = $mem$$index;
3330 int scale = $mem$$scale;
3331 int disp = $mem$$disp;
3332 if (index == -1) {
3333 // Fix up any out-of-range offsets.
3334 assert_different_registers(rscratch1, base);
3335 Address addr = Address(base, disp);
3336 addr = __ legitimize_address(addr, 8, rscratch1);
3337 __ prfm(addr, PSTL1KEEP);
3338 } else {
3339 Register index_reg = as_Register(index);
3340 if (disp == 0) {
3341 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3342 } else {
3343 __ lea(rscratch1, Address(base, disp));
3344 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3345 }
3346 }
3347 %}
3348
3349 // mov encodings
3350
3351 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3352 uint32_t con = (uint32_t)$src$$constant;
3353 Register dst_reg = as_Register($dst$$reg);
3354 if (con == 0) {
3355 __ movw(dst_reg, zr);
3356 } else {
3357 __ movw(dst_reg, con);
3358 }
3359 %}
3360
3361 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3362 Register dst_reg = as_Register($dst$$reg);
3363 uint64_t con = (uint64_t)$src$$constant;
3364 if (con == 0) {
3365 __ mov(dst_reg, zr);
3366 } else {
3367 __ mov(dst_reg, con);
3368 }
3369 %}
3370
3371 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3372 Register dst_reg = as_Register($dst$$reg);
3373 address con = (address)$src$$constant;
3374 if (con == nullptr || con == (address)1) {
3375 ShouldNotReachHere();
3376 } else {
3377 relocInfo::relocType rtype = $src->constant_reloc();
3378 if (rtype == relocInfo::oop_type) {
3379 __ movoop(dst_reg, (jobject)con);
3380 } else if (rtype == relocInfo::metadata_type) {
3381 __ mov_metadata(dst_reg, (Metadata*)con);
3382 } else {
3383 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3384 // load fake address constants using a normal move
3385 if (! __ is_valid_AArch64_address(con) ||
3386 con < (address)(uintptr_t)os::vm_page_size() ||
3387 rtype == relocInfo::none) {
3388 __ mov(dst_reg, con);
3389 } else {
3390 // use shorter adrp/add sequence for external_word relocation
3391 uint64_t offset;
3392 __ adrp(dst_reg, Address(con, rtype), offset);
3393 __ add(dst_reg, dst_reg, offset);
3394 }
3395 }
3396 }
3397 %}
3398
3399 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3400 Register dst_reg = as_Register($dst$$reg);
3401 __ mov(dst_reg, zr);
3402 %}
3403
3404 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3405 Register dst_reg = as_Register($dst$$reg);
3406 __ mov(dst_reg, (uint64_t)1);
3407 %}
3408
3409 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3410 Register dst_reg = as_Register($dst$$reg);
3411 address con = (address)$src$$constant;
3412 if (con == nullptr) {
3413 ShouldNotReachHere();
3414 } else {
3415 relocInfo::relocType rtype = $src->constant_reloc();
3416 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3417 __ set_narrow_oop(dst_reg, (jobject)con);
3418 }
3419 %}
3420
3421 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3422 Register dst_reg = as_Register($dst$$reg);
3423 __ mov(dst_reg, zr);
3424 %}
3425
3426 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 address con = (address)$src$$constant;
3429 if (con == nullptr) {
3430 ShouldNotReachHere();
3431 } else {
3432 relocInfo::relocType rtype = $src->constant_reloc();
3433 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3434 __ set_narrow_klass(dst_reg, (Klass *)con);
3435 }
3436 %}
3437
3438 // arithmetic encodings
3439
3440 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3441 Register dst_reg = as_Register($dst$$reg);
3442 Register src_reg = as_Register($src1$$reg);
3443 int32_t con = (int32_t)$src2$$constant;
3444 // add has primary == 0, subtract has primary == 1
3445 if ($primary) { con = -con; }
3446 if (con < 0) {
3447 __ subw(dst_reg, src_reg, -con);
3448 } else {
3449 __ addw(dst_reg, src_reg, con);
3450 }
3451 %}
3452
3453 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3454 Register dst_reg = as_Register($dst$$reg);
3455 Register src_reg = as_Register($src1$$reg);
3456 int32_t con = (int32_t)$src2$$constant;
3457 // add has primary == 0, subtract has primary == 1
3458 if ($primary) { con = -con; }
3459 if (con < 0) {
3460 __ sub(dst_reg, src_reg, -con);
3461 } else {
3462 __ add(dst_reg, src_reg, con);
3463 }
3464 %}
3465
3466 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3467 Register dst_reg = as_Register($dst$$reg);
3468 Register src1_reg = as_Register($src1$$reg);
3469 Register src2_reg = as_Register($src2$$reg);
3470 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3471 %}
3472
3473 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3474 Register dst_reg = as_Register($dst$$reg);
3475 Register src1_reg = as_Register($src1$$reg);
3476 Register src2_reg = as_Register($src2$$reg);
3477 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3478 %}
3479
3480 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3481 Register dst_reg = as_Register($dst$$reg);
3482 Register src1_reg = as_Register($src1$$reg);
3483 Register src2_reg = as_Register($src2$$reg);
3484 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3485 %}
3486
3487 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3488 Register dst_reg = as_Register($dst$$reg);
3489 Register src1_reg = as_Register($src1$$reg);
3490 Register src2_reg = as_Register($src2$$reg);
3491 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3492 %}
3493
3494 // compare instruction encodings
3495
3496 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3497 Register reg1 = as_Register($src1$$reg);
3498 Register reg2 = as_Register($src2$$reg);
3499 __ cmpw(reg1, reg2);
3500 %}
3501
3502 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3503 Register reg = as_Register($src1$$reg);
3504 int32_t val = $src2$$constant;
3505 if (val >= 0) {
3506 __ subsw(zr, reg, val);
3507 } else {
3508 __ addsw(zr, reg, -val);
3509 }
3510 %}
3511
3512 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3513 Register reg1 = as_Register($src1$$reg);
3514 uint32_t val = (uint32_t)$src2$$constant;
3515 __ movw(rscratch1, val);
3516 __ cmpw(reg1, rscratch1);
3517 %}
3518
3519 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3520 Register reg1 = as_Register($src1$$reg);
3521 Register reg2 = as_Register($src2$$reg);
3522 __ cmp(reg1, reg2);
3523 %}
3524
3525 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3526 Register reg = as_Register($src1$$reg);
3527 int64_t val = $src2$$constant;
3528 if (val >= 0) {
3529 __ subs(zr, reg, val);
3530 } else if (val != -val) {
3531 __ adds(zr, reg, -val);
3532 } else {
3533 // aargh, Long.MIN_VALUE is a special case
3534 __ orr(rscratch1, zr, (uint64_t)val);
3535 __ subs(zr, reg, rscratch1);
3536 }
3537 %}
3538
3539 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3540 Register reg1 = as_Register($src1$$reg);
3541 uint64_t val = (uint64_t)$src2$$constant;
3542 __ mov(rscratch1, val);
3543 __ cmp(reg1, rscratch1);
3544 %}
3545
3546 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3547 Register reg1 = as_Register($src1$$reg);
3548 Register reg2 = as_Register($src2$$reg);
3549 __ cmp(reg1, reg2);
3550 %}
3551
3552 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3553 Register reg1 = as_Register($src1$$reg);
3554 Register reg2 = as_Register($src2$$reg);
3555 __ cmpw(reg1, reg2);
3556 %}
3557
3558 enc_class aarch64_enc_testp(iRegP src) %{
3559 Register reg = as_Register($src$$reg);
3560 __ cmp(reg, zr);
3561 %}
3562
3563 enc_class aarch64_enc_testn(iRegN src) %{
3564 Register reg = as_Register($src$$reg);
3565 __ cmpw(reg, zr);
3566 %}
3567
3568 enc_class aarch64_enc_b(label lbl) %{
3569 Label *L = $lbl$$label;
3570 __ b(*L);
3571 %}
3572
3573 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3574 Label *L = $lbl$$label;
3575 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3576 %}
3577
3578 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3579 Label *L = $lbl$$label;
3580 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3581 %}
3582
3583 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3584 %{
3585 Register sub_reg = as_Register($sub$$reg);
3586 Register super_reg = as_Register($super$$reg);
3587 Register temp_reg = as_Register($temp$$reg);
3588 Register result_reg = as_Register($result$$reg);
3589
3590 Label miss;
3591 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3592 nullptr, &miss,
3593 /*set_cond_codes:*/ true);
3594 if ($primary) {
3595 __ mov(result_reg, zr);
3596 }
3597 __ bind(miss);
3598 %}
3599
3600 enc_class aarch64_enc_java_static_call(method meth) %{
3601 address addr = (address)$meth$$method;
3602 address call;
3603 if (!_method) {
3604 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3605 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3606 if (call == nullptr) {
3607 ciEnv::current()->record_failure("CodeCache is full");
3608 return;
3609 }
3610 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3611 // The NOP here is purely to ensure that eliding a call to
3612 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3613 __ nop();
3614 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3615 } else {
3616 int method_index = resolved_method_index(masm);
3617 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3618 : static_call_Relocation::spec(method_index);
3619 call = __ trampoline_call(Address(addr, rspec));
3620 if (call == nullptr) {
3621 ciEnv::current()->record_failure("CodeCache is full");
3622 return;
3623 }
3624 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3625 // Calls of the same statically bound method can share
3626 // a stub to the interpreter.
3627 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3628 } else {
3629 // Emit stub for static call
3630 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3631 if (stub == nullptr) {
3632 ciEnv::current()->record_failure("CodeCache is full");
3633 return;
3634 }
3635 }
3636 }
3637
3638 __ post_call_nop();
3639
3640 // Only non uncommon_trap calls need to reinitialize ptrue.
3641 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3642 __ reinitialize_ptrue();
3643 }
3644 %}
3645
3646 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3647 int method_index = resolved_method_index(masm);
3648 address call = __ ic_call((address)$meth$$method, method_index);
3649 if (call == nullptr) {
3650 ciEnv::current()->record_failure("CodeCache is full");
3651 return;
3652 }
3653 __ post_call_nop();
3654 if (Compile::current()->max_vector_size() > 0) {
3655 __ reinitialize_ptrue();
3656 }
3657 %}
3658
3659 enc_class aarch64_enc_call_epilog() %{
3660 if (VerifyStackAtCalls) {
3661 // Check that stack depth is unchanged: find majik cookie on stack
3662 __ call_Unimplemented();
3663 }
3664 %}
3665
3666 enc_class aarch64_enc_java_to_runtime(method meth) %{
3667 // some calls to generated routines (arraycopy code) are scheduled
3668 // by C2 as runtime calls. if so we can call them using a br (they
3669 // will be in a reachable segment) otherwise we have to use a blr
3670 // which loads the absolute address into a register.
3671 address entry = (address)$meth$$method;
3672 CodeBlob *cb = CodeCache::find_blob(entry);
3673 if (cb) {
3674 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3675 if (call == nullptr) {
3676 ciEnv::current()->record_failure("CodeCache is full");
3677 return;
3678 }
3679 __ post_call_nop();
3680 } else {
3681 Label retaddr;
3682 // Make the anchor frame walkable
3683 __ adr(rscratch2, retaddr);
3684 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3685 __ lea(rscratch1, RuntimeAddress(entry));
3686 __ blr(rscratch1);
3687 __ bind(retaddr);
3688 __ post_call_nop();
3689 }
3690 if (Compile::current()->max_vector_size() > 0) {
3691 __ reinitialize_ptrue();
3692 }
3693 %}
3694
3695 enc_class aarch64_enc_rethrow() %{
3696 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3697 %}
3698
3699 enc_class aarch64_enc_ret() %{
3700 #ifdef ASSERT
3701 if (Compile::current()->max_vector_size() > 0) {
3702 __ verify_ptrue();
3703 }
3704 #endif
3705 __ ret(lr);
3706 %}
3707
3708 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3709 Register target_reg = as_Register($jump_target$$reg);
3710 __ br(target_reg);
3711 %}
3712
3713 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3714 Register target_reg = as_Register($jump_target$$reg);
3715 // exception oop should be in r0
3716 // ret addr has been popped into lr
3717 // callee expects it in r3
3718 __ mov(r3, lr);
3719 __ br(target_reg);
3720 %}
3721
3722 %}
3723
3724 //----------FRAME--------------------------------------------------------------
3725 // Definition of frame structure and management information.
3726 //
3727 // S T A C K L A Y O U T Allocators stack-slot number
3728 // | (to get allocators register number
3729 // G Owned by | | v add OptoReg::stack0())
3730 // r CALLER | |
3731 // o | +--------+ pad to even-align allocators stack-slot
3732 // w V | pad0 | numbers; owned by CALLER
3733 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3734 // h ^ | in | 5
3735 // | | args | 4 Holes in incoming args owned by SELF
3736 // | | | | 3
3737 // | | +--------+
3738 // V | | old out| Empty on Intel, window on Sparc
3739 // | old |preserve| Must be even aligned.
3740 // | SP-+--------+----> Matcher::_old_SP, even aligned
3741 // | | in | 3 area for Intel ret address
3742 // Owned by |preserve| Empty on Sparc.
3743 // SELF +--------+
3744 // | | pad2 | 2 pad to align old SP
3745 // | +--------+ 1
3746 // | | locks | 0
3747 // | +--------+----> OptoReg::stack0(), even aligned
3748 // | | pad1 | 11 pad to align new SP
3749 // | +--------+
3750 // | | | 10
3751 // | | spills | 9 spills
3752 // V | | 8 (pad0 slot for callee)
3753 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3754 // ^ | out | 7
3755 // | | args | 6 Holes in outgoing args owned by CALLEE
3756 // Owned by +--------+
3757 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3758 // | new |preserve| Must be even-aligned.
3759 // | SP-+--------+----> Matcher::_new_SP, even aligned
3760 // | | |
3761 //
3762 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3763 // known from SELF's arguments and the Java calling convention.
3764 // Region 6-7 is determined per call site.
3765 // Note 2: If the calling convention leaves holes in the incoming argument
3766 // area, those holes are owned by SELF. Holes in the outgoing area
3767 // are owned by the CALLEE. Holes should not be necessary in the
3768 // incoming area, as the Java calling convention is completely under
3769 // the control of the AD file. Doubles can be sorted and packed to
3770 // avoid holes. Holes in the outgoing arguments may be necessary for
3771 // varargs C calling conventions.
3772 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3773 // even aligned with pad0 as needed.
3774 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3775 // (the latter is true on Intel but is it false on AArch64?)
3776 // region 6-11 is even aligned; it may be padded out more so that
3777 // the region from SP to FP meets the minimum stack alignment.
3778 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3779 // alignment. Region 11, pad1, may be dynamically extended so that
3780 // SP meets the minimum alignment.
3781
3782 frame %{
3783 // These three registers define part of the calling convention
3784 // between compiled code and the interpreter.
3785
3786 // Inline Cache Register or Method for I2C.
3787 inline_cache_reg(R12);
3788
3789 // Number of stack slots consumed by locking an object
3790 sync_stack_slots(2);
3791
3792 // Compiled code's Frame Pointer
3793 frame_pointer(R31);
3794
3795 // Stack alignment requirement
3796 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3797
3798 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3799 // for calls to C. Supports the var-args backing area for register parms.
3800 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3801
3802 // The after-PROLOG location of the return address. Location of
3803 // return address specifies a type (REG or STACK) and a number
3804 // representing the register number (i.e. - use a register name) or
3805 // stack slot.
3806 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3807 // Otherwise, it is above the locks and verification slot and alignment word
3808 // TODO this may well be correct but need to check why that - 2 is there
3809 // ppc port uses 0 but we definitely need to allow for fixed_slots
3810 // which folds in the space used for monitors
3811 return_addr(STACK - 2 +
3812 align_up((Compile::current()->in_preserve_stack_slots() +
3813 Compile::current()->fixed_slots()),
3814 stack_alignment_in_slots()));
3815
3816 // Location of compiled Java return values. Same as C for now.
3817 return_value
3818 %{
3819 // TODO do we allow ideal_reg == Op_RegN???
3820 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3821 "only return normal values");
3822
3823 static const int lo[Op_RegL + 1] = { // enum name
3824 0, // Op_Node
3825 0, // Op_Set
3826 R0_num, // Op_RegN
3827 R0_num, // Op_RegI
3828 R0_num, // Op_RegP
3829 V0_num, // Op_RegF
3830 V0_num, // Op_RegD
3831 R0_num // Op_RegL
3832 };
3833
3834 static const int hi[Op_RegL + 1] = { // enum name
3835 0, // Op_Node
3836 0, // Op_Set
3837 OptoReg::Bad, // Op_RegN
3838 OptoReg::Bad, // Op_RegI
3839 R0_H_num, // Op_RegP
3840 OptoReg::Bad, // Op_RegF
3841 V0_H_num, // Op_RegD
3842 R0_H_num // Op_RegL
3843 };
3844
3845 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3846 %}
3847 %}
3848
3849 //----------ATTRIBUTES---------------------------------------------------------
3850 //----------Operand Attributes-------------------------------------------------
3851 op_attrib op_cost(1); // Required cost attribute
3852
3853 //----------Instruction Attributes---------------------------------------------
3854 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3855 ins_attrib ins_size(32); // Required size attribute (in bits)
3856 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3857 // a non-matching short branch variant
3858 // of some long branch?
3859 ins_attrib ins_alignment(4); // Required alignment attribute (must
3860 // be a power of 2) specifies the
3861 // alignment that some part of the
3862 // instruction (not necessarily the
3863 // start) requires. If > 1, a
3864 // compute_padding() function must be
3865 // provided for the instruction
3866
3867 // Whether this node is expanded during code emission into a sequence of
3868 // instructions and the first instruction can perform an implicit null check.
3869 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3870
3871 //----------OPERANDS-----------------------------------------------------------
3872 // Operand definitions must precede instruction definitions for correct parsing
3873 // in the ADLC because operands constitute user defined types which are used in
3874 // instruction definitions.
3875
3876 //----------Simple Operands----------------------------------------------------
3877
3878 // Integer operands 32 bit
3879 // 32 bit immediate
3880 operand immI()
3881 %{
3882 match(ConI);
3883
3884 op_cost(0);
3885 format %{ %}
3886 interface(CONST_INTER);
3887 %}
3888
3889 // 32 bit zero
3890 operand immI0()
3891 %{
3892 predicate(n->get_int() == 0);
3893 match(ConI);
3894
3895 op_cost(0);
3896 format %{ %}
3897 interface(CONST_INTER);
3898 %}
3899
3900 // 32 bit unit increment
3901 operand immI_1()
3902 %{
3903 predicate(n->get_int() == 1);
3904 match(ConI);
3905
3906 op_cost(0);
3907 format %{ %}
3908 interface(CONST_INTER);
3909 %}
3910
3911 // 32 bit unit decrement
3912 operand immI_M1()
3913 %{
3914 predicate(n->get_int() == -1);
3915 match(ConI);
3916
3917 op_cost(0);
3918 format %{ %}
3919 interface(CONST_INTER);
3920 %}
3921
3922 // Shift values for add/sub extension shift
3923 operand immIExt()
3924 %{
3925 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3926 match(ConI);
3927
3928 op_cost(0);
3929 format %{ %}
3930 interface(CONST_INTER);
3931 %}
3932
3933 operand immI_gt_1()
3934 %{
3935 predicate(n->get_int() > 1);
3936 match(ConI);
3937
3938 op_cost(0);
3939 format %{ %}
3940 interface(CONST_INTER);
3941 %}
3942
3943 operand immI_le_4()
3944 %{
3945 predicate(n->get_int() <= 4);
3946 match(ConI);
3947
3948 op_cost(0);
3949 format %{ %}
3950 interface(CONST_INTER);
3951 %}
3952
3953 operand immI_16()
3954 %{
3955 predicate(n->get_int() == 16);
3956 match(ConI);
3957
3958 op_cost(0);
3959 format %{ %}
3960 interface(CONST_INTER);
3961 %}
3962
3963 operand immI_24()
3964 %{
3965 predicate(n->get_int() == 24);
3966 match(ConI);
3967
3968 op_cost(0);
3969 format %{ %}
3970 interface(CONST_INTER);
3971 %}
3972
3973 operand immI_32()
3974 %{
3975 predicate(n->get_int() == 32);
3976 match(ConI);
3977
3978 op_cost(0);
3979 format %{ %}
3980 interface(CONST_INTER);
3981 %}
3982
3983 operand immI_48()
3984 %{
3985 predicate(n->get_int() == 48);
3986 match(ConI);
3987
3988 op_cost(0);
3989 format %{ %}
3990 interface(CONST_INTER);
3991 %}
3992
3993 operand immI_56()
3994 %{
3995 predicate(n->get_int() == 56);
3996 match(ConI);
3997
3998 op_cost(0);
3999 format %{ %}
4000 interface(CONST_INTER);
4001 %}
4002
4003 operand immI_255()
4004 %{
4005 predicate(n->get_int() == 255);
4006 match(ConI);
4007
4008 op_cost(0);
4009 format %{ %}
4010 interface(CONST_INTER);
4011 %}
4012
4013 operand immI_65535()
4014 %{
4015 predicate(n->get_int() == 65535);
4016 match(ConI);
4017
4018 op_cost(0);
4019 format %{ %}
4020 interface(CONST_INTER);
4021 %}
4022
4023 operand immI_positive()
4024 %{
4025 predicate(n->get_int() > 0);
4026 match(ConI);
4027
4028 op_cost(0);
4029 format %{ %}
4030 interface(CONST_INTER);
4031 %}
4032
4033 // BoolTest condition for signed compare
4034 operand immI_cmp_cond()
4035 %{
4036 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4037 match(ConI);
4038
4039 op_cost(0);
4040 format %{ %}
4041 interface(CONST_INTER);
4042 %}
4043
4044 // BoolTest condition for unsigned compare
4045 operand immI_cmpU_cond()
4046 %{
4047 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4048 match(ConI);
4049
4050 op_cost(0);
4051 format %{ %}
4052 interface(CONST_INTER);
4053 %}
4054
4055 operand immL_255()
4056 %{
4057 predicate(n->get_long() == 255L);
4058 match(ConL);
4059
4060 op_cost(0);
4061 format %{ %}
4062 interface(CONST_INTER);
4063 %}
4064
4065 operand immL_65535()
4066 %{
4067 predicate(n->get_long() == 65535L);
4068 match(ConL);
4069
4070 op_cost(0);
4071 format %{ %}
4072 interface(CONST_INTER);
4073 %}
4074
4075 operand immL_4294967295()
4076 %{
4077 predicate(n->get_long() == 4294967295L);
4078 match(ConL);
4079
4080 op_cost(0);
4081 format %{ %}
4082 interface(CONST_INTER);
4083 %}
4084
4085 operand immL_bitmask()
4086 %{
4087 predicate((n->get_long() != 0)
4088 && ((n->get_long() & 0xc000000000000000l) == 0)
4089 && is_power_of_2(n->get_long() + 1));
4090 match(ConL);
4091
4092 op_cost(0);
4093 format %{ %}
4094 interface(CONST_INTER);
4095 %}
4096
4097 operand immI_bitmask()
4098 %{
4099 predicate((n->get_int() != 0)
4100 && ((n->get_int() & 0xc0000000) == 0)
4101 && is_power_of_2(n->get_int() + 1));
4102 match(ConI);
4103
4104 op_cost(0);
4105 format %{ %}
4106 interface(CONST_INTER);
4107 %}
4108
4109 operand immL_positive_bitmaskI()
4110 %{
4111 predicate((n->get_long() != 0)
4112 && ((julong)n->get_long() < 0x80000000ULL)
4113 && is_power_of_2(n->get_long() + 1));
4114 match(ConL);
4115
4116 op_cost(0);
4117 format %{ %}
4118 interface(CONST_INTER);
4119 %}
4120
4121 // Scale values for scaled offset addressing modes (up to long but not quad)
4122 operand immIScale()
4123 %{
4124 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4125 match(ConI);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 // 5 bit signed integer
4133 operand immI5()
4134 %{
4135 predicate(Assembler::is_simm(n->get_int(), 5));
4136 match(ConI);
4137
4138 op_cost(0);
4139 format %{ %}
4140 interface(CONST_INTER);
4141 %}
4142
4143 // 7 bit unsigned integer
4144 operand immIU7()
4145 %{
4146 predicate(Assembler::is_uimm(n->get_int(), 7));
4147 match(ConI);
4148
4149 op_cost(0);
4150 format %{ %}
4151 interface(CONST_INTER);
4152 %}
4153
4154 // Offset for scaled or unscaled immediate loads and stores
4155 operand immIOffset()
4156 %{
4157 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4158 match(ConI);
4159
4160 op_cost(0);
4161 format %{ %}
4162 interface(CONST_INTER);
4163 %}
4164
4165 operand immIOffset1()
4166 %{
4167 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4168 match(ConI);
4169
4170 op_cost(0);
4171 format %{ %}
4172 interface(CONST_INTER);
4173 %}
4174
4175 operand immIOffset2()
4176 %{
4177 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4178 match(ConI);
4179
4180 op_cost(0);
4181 format %{ %}
4182 interface(CONST_INTER);
4183 %}
4184
4185 operand immIOffset4()
4186 %{
4187 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4188 match(ConI);
4189
4190 op_cost(0);
4191 format %{ %}
4192 interface(CONST_INTER);
4193 %}
4194
4195 operand immIOffset8()
4196 %{
4197 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4198 match(ConI);
4199
4200 op_cost(0);
4201 format %{ %}
4202 interface(CONST_INTER);
4203 %}
4204
4205 operand immIOffset16()
4206 %{
4207 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4208 match(ConI);
4209
4210 op_cost(0);
4211 format %{ %}
4212 interface(CONST_INTER);
4213 %}
4214
4215 operand immLOffset()
4216 %{
4217 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4218 match(ConL);
4219
4220 op_cost(0);
4221 format %{ %}
4222 interface(CONST_INTER);
4223 %}
4224
4225 operand immLoffset1()
4226 %{
4227 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4228 match(ConL);
4229
4230 op_cost(0);
4231 format %{ %}
4232 interface(CONST_INTER);
4233 %}
4234
4235 operand immLoffset2()
4236 %{
4237 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4238 match(ConL);
4239
4240 op_cost(0);
4241 format %{ %}
4242 interface(CONST_INTER);
4243 %}
4244
4245 operand immLoffset4()
4246 %{
4247 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4248 match(ConL);
4249
4250 op_cost(0);
4251 format %{ %}
4252 interface(CONST_INTER);
4253 %}
4254
4255 operand immLoffset8()
4256 %{
4257 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4258 match(ConL);
4259
4260 op_cost(0);
4261 format %{ %}
4262 interface(CONST_INTER);
4263 %}
4264
4265 operand immLoffset16()
4266 %{
4267 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4268 match(ConL);
4269
4270 op_cost(0);
4271 format %{ %}
4272 interface(CONST_INTER);
4273 %}
4274
4275 // 5 bit signed long integer
4276 operand immL5()
4277 %{
4278 predicate(Assembler::is_simm(n->get_long(), 5));
4279 match(ConL);
4280
4281 op_cost(0);
4282 format %{ %}
4283 interface(CONST_INTER);
4284 %}
4285
4286 // 7 bit unsigned long integer
4287 operand immLU7()
4288 %{
4289 predicate(Assembler::is_uimm(n->get_long(), 7));
4290 match(ConL);
4291
4292 op_cost(0);
4293 format %{ %}
4294 interface(CONST_INTER);
4295 %}
4296
4297 // 8 bit signed value.
4298 operand immI8()
4299 %{
4300 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4301 match(ConI);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 // 8 bit signed value (simm8), or #simm8 LSL 8.
4309 operand immIDupV()
4310 %{
4311 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4312 match(ConI);
4313
4314 op_cost(0);
4315 format %{ %}
4316 interface(CONST_INTER);
4317 %}
4318
4319 // 8 bit signed value (simm8), or #simm8 LSL 8.
4320 operand immLDupV()
4321 %{
4322 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4323 match(ConL);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 // 8 bit signed value (simm8), or #simm8 LSL 8.
4331 operand immHDupV()
4332 %{
4333 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4334 match(ConH);
4335
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 // 8 bit integer valid for vector add sub immediate
4342 operand immBAddSubV()
4343 %{
4344 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4345 match(ConI);
4346
4347 op_cost(0);
4348 format %{ %}
4349 interface(CONST_INTER);
4350 %}
4351
4352 // 32 bit integer valid for add sub immediate
4353 operand immIAddSub()
4354 %{
4355 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4356 match(ConI);
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 // 32 bit integer valid for vector add sub immediate
4363 operand immIAddSubV()
4364 %{
4365 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4366 match(ConI);
4367
4368 op_cost(0);
4369 format %{ %}
4370 interface(CONST_INTER);
4371 %}
4372
4373 // 32 bit unsigned integer valid for logical immediate
4374
4375 operand immBLog()
4376 %{
4377 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4378 match(ConI);
4379
4380 op_cost(0);
4381 format %{ %}
4382 interface(CONST_INTER);
4383 %}
4384
4385 operand immSLog()
4386 %{
4387 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4388 match(ConI);
4389
4390 op_cost(0);
4391 format %{ %}
4392 interface(CONST_INTER);
4393 %}
4394
4395 operand immILog()
4396 %{
4397 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4398 match(ConI);
4399
4400 op_cost(0);
4401 format %{ %}
4402 interface(CONST_INTER);
4403 %}
4404
4405 // Integer operands 64 bit
4406 // 64 bit immediate
4407 operand immL()
4408 %{
4409 match(ConL);
4410
4411 op_cost(0);
4412 format %{ %}
4413 interface(CONST_INTER);
4414 %}
4415
4416 // 64 bit zero
4417 operand immL0()
4418 %{
4419 predicate(n->get_long() == 0);
4420 match(ConL);
4421
4422 op_cost(0);
4423 format %{ %}
4424 interface(CONST_INTER);
4425 %}
4426
4427 // 64 bit unit decrement
4428 operand immL_M1()
4429 %{
4430 predicate(n->get_long() == -1);
4431 match(ConL);
4432
4433 op_cost(0);
4434 format %{ %}
4435 interface(CONST_INTER);
4436 %}
4437
4438 // 64 bit integer valid for add sub immediate
4439 operand immLAddSub()
4440 %{
4441 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4442 match(ConL);
4443 op_cost(0);
4444 format %{ %}
4445 interface(CONST_INTER);
4446 %}
4447
4448 // 64 bit integer valid for addv subv immediate
4449 operand immLAddSubV()
4450 %{
4451 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4452 match(ConL);
4453
4454 op_cost(0);
4455 format %{ %}
4456 interface(CONST_INTER);
4457 %}
4458
4459 // 64 bit integer valid for logical immediate
4460 operand immLLog()
4461 %{
4462 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4463 match(ConL);
4464 op_cost(0);
4465 format %{ %}
4466 interface(CONST_INTER);
4467 %}
4468
4469 // Long Immediate: low 32-bit mask
4470 operand immL_32bits()
4471 %{
4472 predicate(n->get_long() == 0xFFFFFFFFL);
4473 match(ConL);
4474 op_cost(0);
4475 format %{ %}
4476 interface(CONST_INTER);
4477 %}
4478
4479 // Pointer operands
4480 // Pointer Immediate
4481 operand immP()
4482 %{
4483 match(ConP);
4484
4485 op_cost(0);
4486 format %{ %}
4487 interface(CONST_INTER);
4488 %}
4489
4490 // nullptr Pointer Immediate
4491 operand immP0()
4492 %{
4493 predicate(n->get_ptr() == 0);
4494 match(ConP);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // Pointer Immediate One
4502 // this is used in object initialization (initial object header)
4503 operand immP_1()
4504 %{
4505 predicate(n->get_ptr() == 1);
4506 match(ConP);
4507
4508 op_cost(0);
4509 format %{ %}
4510 interface(CONST_INTER);
4511 %}
4512
4513 // AOT Runtime Constants Address
4514 operand immAOTRuntimeConstantsAddress()
4515 %{
4516 // Check if the address is in the range of AOT Runtime Constants
4517 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4518 match(ConP);
4519
4520 op_cost(0);
4521 format %{ %}
4522 interface(CONST_INTER);
4523 %}
4524
4525 // Float and Double operands
4526 // Double Immediate
4527 operand immD()
4528 %{
4529 match(ConD);
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 // Double Immediate: +0.0d
4536 operand immD0()
4537 %{
4538 predicate(jlong_cast(n->getd()) == 0);
4539 match(ConD);
4540
4541 op_cost(0);
4542 format %{ %}
4543 interface(CONST_INTER);
4544 %}
4545
4546 // constant 'double +0.0'.
4547 operand immDPacked()
4548 %{
4549 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4550 match(ConD);
4551 op_cost(0);
4552 format %{ %}
4553 interface(CONST_INTER);
4554 %}
4555
4556 // Float Immediate
4557 operand immF()
4558 %{
4559 match(ConF);
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // Float Immediate: +0.0f.
4566 operand immF0()
4567 %{
4568 predicate(jint_cast(n->getf()) == 0);
4569 match(ConF);
4570
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // Half Float (FP16) Immediate
4577 operand immH()
4578 %{
4579 match(ConH);
4580 op_cost(0);
4581 format %{ %}
4582 interface(CONST_INTER);
4583 %}
4584
4585 //
4586 operand immFPacked()
4587 %{
4588 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4589 match(ConF);
4590 op_cost(0);
4591 format %{ %}
4592 interface(CONST_INTER);
4593 %}
4594
4595 // Narrow pointer operands
4596 // Narrow Pointer Immediate
4597 operand immN()
4598 %{
4599 match(ConN);
4600
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Narrow nullptr Pointer Immediate
4607 operand immN0()
4608 %{
4609 predicate(n->get_narrowcon() == 0);
4610 match(ConN);
4611
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 operand immNKlass()
4618 %{
4619 match(ConNKlass);
4620
4621 op_cost(0);
4622 format %{ %}
4623 interface(CONST_INTER);
4624 %}
4625
4626 // Integer 32 bit Register Operands
4627 // Integer 32 bitRegister (excludes SP)
4628 operand iRegI()
4629 %{
4630 constraint(ALLOC_IN_RC(any_reg32));
4631 match(RegI);
4632 match(iRegINoSp);
4633 op_cost(0);
4634 format %{ %}
4635 interface(REG_INTER);
4636 %}
4637
4638 // Integer 32 bit Register not Special
4639 operand iRegINoSp()
4640 %{
4641 constraint(ALLOC_IN_RC(no_special_reg32));
4642 match(RegI);
4643 op_cost(0);
4644 format %{ %}
4645 interface(REG_INTER);
4646 %}
4647
4648 // Integer 64 bit Register Operands
4649 // Integer 64 bit Register (includes SP)
4650 operand iRegL()
4651 %{
4652 constraint(ALLOC_IN_RC(any_reg));
4653 match(RegL);
4654 match(iRegLNoSp);
4655 op_cost(0);
4656 format %{ %}
4657 interface(REG_INTER);
4658 %}
4659
4660 // Integer 64 bit Register not Special
4661 operand iRegLNoSp()
4662 %{
4663 constraint(ALLOC_IN_RC(no_special_reg));
4664 match(RegL);
4665 match(iRegL_R0);
4666 format %{ %}
4667 interface(REG_INTER);
4668 %}
4669
4670 // Pointer Register Operands
4671 // Pointer Register
4672 operand iRegP()
4673 %{
4674 constraint(ALLOC_IN_RC(ptr_reg));
4675 match(RegP);
4676 match(iRegPNoSp);
4677 match(iRegP_R0);
4678 //match(iRegP_R2);
4679 //match(iRegP_R4);
4680 match(iRegP_R5);
4681 match(thread_RegP);
4682 op_cost(0);
4683 format %{ %}
4684 interface(REG_INTER);
4685 %}
4686
4687 // Pointer 64 bit Register not Special
4688 operand iRegPNoSp()
4689 %{
4690 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4691 match(RegP);
4692 // match(iRegP);
4693 // match(iRegP_R0);
4694 // match(iRegP_R2);
4695 // match(iRegP_R4);
4696 // match(iRegP_R5);
4697 // match(thread_RegP);
4698 op_cost(0);
4699 format %{ %}
4700 interface(REG_INTER);
4701 %}
4702
4703 // This operand is not allowed to use rfp even if
4704 // rfp is not used to hold the frame pointer.
4705 operand iRegPNoSpNoRfp()
4706 %{
4707 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4708 match(RegP);
4709 match(iRegPNoSp);
4710 op_cost(0);
4711 format %{ %}
4712 interface(REG_INTER);
4713 %}
4714
4715 // Pointer 64 bit Register R0 only
4716 operand iRegP_R0()
4717 %{
4718 constraint(ALLOC_IN_RC(r0_reg));
4719 match(RegP);
4720 // match(iRegP);
4721 match(iRegPNoSp);
4722 op_cost(0);
4723 format %{ %}
4724 interface(REG_INTER);
4725 %}
4726
4727 // Pointer 64 bit Register R1 only
4728 operand iRegP_R1()
4729 %{
4730 constraint(ALLOC_IN_RC(r1_reg));
4731 match(RegP);
4732 // match(iRegP);
4733 match(iRegPNoSp);
4734 op_cost(0);
4735 format %{ %}
4736 interface(REG_INTER);
4737 %}
4738
4739 // Pointer 64 bit Register R2 only
4740 operand iRegP_R2()
4741 %{
4742 constraint(ALLOC_IN_RC(r2_reg));
4743 match(RegP);
4744 // match(iRegP);
4745 match(iRegPNoSp);
4746 op_cost(0);
4747 format %{ %}
4748 interface(REG_INTER);
4749 %}
4750
4751 // Pointer 64 bit Register R3 only
4752 operand iRegP_R3()
4753 %{
4754 constraint(ALLOC_IN_RC(r3_reg));
4755 match(RegP);
4756 // match(iRegP);
4757 match(iRegPNoSp);
4758 op_cost(0);
4759 format %{ %}
4760 interface(REG_INTER);
4761 %}
4762
4763 // Pointer 64 bit Register R4 only
4764 operand iRegP_R4()
4765 %{
4766 constraint(ALLOC_IN_RC(r4_reg));
4767 match(RegP);
4768 // match(iRegP);
4769 match(iRegPNoSp);
4770 op_cost(0);
4771 format %{ %}
4772 interface(REG_INTER);
4773 %}
4774
4775 // Pointer 64 bit Register R5 only
4776 operand iRegP_R5()
4777 %{
4778 constraint(ALLOC_IN_RC(r5_reg));
4779 match(RegP);
4780 // match(iRegP);
4781 match(iRegPNoSp);
4782 op_cost(0);
4783 format %{ %}
4784 interface(REG_INTER);
4785 %}
4786
4787 // Pointer 64 bit Register R10 only
4788 operand iRegP_R10()
4789 %{
4790 constraint(ALLOC_IN_RC(r10_reg));
4791 match(RegP);
4792 // match(iRegP);
4793 match(iRegPNoSp);
4794 op_cost(0);
4795 format %{ %}
4796 interface(REG_INTER);
4797 %}
4798
4799 // Long 64 bit Register R0 only
4800 operand iRegL_R0()
4801 %{
4802 constraint(ALLOC_IN_RC(r0_reg));
4803 match(RegL);
4804 match(iRegLNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Long 64 bit Register R11 only
4811 operand iRegL_R11()
4812 %{
4813 constraint(ALLOC_IN_RC(r11_reg));
4814 match(RegL);
4815 match(iRegLNoSp);
4816 op_cost(0);
4817 format %{ %}
4818 interface(REG_INTER);
4819 %}
4820
4821 // Register R0 only
4822 operand iRegI_R0()
4823 %{
4824 constraint(ALLOC_IN_RC(int_r0_reg));
4825 match(RegI);
4826 match(iRegINoSp);
4827 op_cost(0);
4828 format %{ %}
4829 interface(REG_INTER);
4830 %}
4831
4832 // Register R2 only
4833 operand iRegI_R2()
4834 %{
4835 constraint(ALLOC_IN_RC(int_r2_reg));
4836 match(RegI);
4837 match(iRegINoSp);
4838 op_cost(0);
4839 format %{ %}
4840 interface(REG_INTER);
4841 %}
4842
4843 // Register R3 only
4844 operand iRegI_R3()
4845 %{
4846 constraint(ALLOC_IN_RC(int_r3_reg));
4847 match(RegI);
4848 match(iRegINoSp);
4849 op_cost(0);
4850 format %{ %}
4851 interface(REG_INTER);
4852 %}
4853
4854
4855 // Register R4 only
4856 operand iRegI_R4()
4857 %{
4858 constraint(ALLOC_IN_RC(int_r4_reg));
4859 match(RegI);
4860 match(iRegINoSp);
4861 op_cost(0);
4862 format %{ %}
4863 interface(REG_INTER);
4864 %}
4865
4866
4867 // Pointer Register Operands
4868 // Narrow Pointer Register
4869 operand iRegN()
4870 %{
4871 constraint(ALLOC_IN_RC(any_reg32));
4872 match(RegN);
4873 match(iRegNNoSp);
4874 op_cost(0);
4875 format %{ %}
4876 interface(REG_INTER);
4877 %}
4878
4879 // Integer 64 bit Register not Special
4880 operand iRegNNoSp()
4881 %{
4882 constraint(ALLOC_IN_RC(no_special_reg32));
4883 match(RegN);
4884 op_cost(0);
4885 format %{ %}
4886 interface(REG_INTER);
4887 %}
4888
4889 // Float Register
4890 // Float register operands
4891 operand vRegF()
4892 %{
4893 constraint(ALLOC_IN_RC(float_reg));
4894 match(RegF);
4895
4896 op_cost(0);
4897 format %{ %}
4898 interface(REG_INTER);
4899 %}
4900
4901 // Double Register
4902 // Double register operands
4903 operand vRegD()
4904 %{
4905 constraint(ALLOC_IN_RC(double_reg));
4906 match(RegD);
4907
4908 op_cost(0);
4909 format %{ %}
4910 interface(REG_INTER);
4911 %}
4912
4913 // Generic vector class. This will be used for
4914 // all vector operands, including NEON and SVE.
4915 operand vReg()
4916 %{
4917 constraint(ALLOC_IN_RC(dynamic));
4918 match(VecA);
4919 match(VecD);
4920 match(VecX);
4921
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 operand vReg_V10()
4928 %{
4929 constraint(ALLOC_IN_RC(v10_veca_reg));
4930 match(vReg);
4931
4932 op_cost(0);
4933 format %{ %}
4934 interface(REG_INTER);
4935 %}
4936
4937 operand vReg_V11()
4938 %{
4939 constraint(ALLOC_IN_RC(v11_veca_reg));
4940 match(vReg);
4941
4942 op_cost(0);
4943 format %{ %}
4944 interface(REG_INTER);
4945 %}
4946
4947 operand vReg_V12()
4948 %{
4949 constraint(ALLOC_IN_RC(v12_veca_reg));
4950 match(vReg);
4951
4952 op_cost(0);
4953 format %{ %}
4954 interface(REG_INTER);
4955 %}
4956
4957 operand vReg_V13()
4958 %{
4959 constraint(ALLOC_IN_RC(v13_veca_reg));
4960 match(vReg);
4961
4962 op_cost(0);
4963 format %{ %}
4964 interface(REG_INTER);
4965 %}
4966
4967 operand vReg_V17()
4968 %{
4969 constraint(ALLOC_IN_RC(v17_veca_reg));
4970 match(vReg);
4971
4972 op_cost(0);
4973 format %{ %}
4974 interface(REG_INTER);
4975 %}
4976
4977 operand vReg_V18()
4978 %{
4979 constraint(ALLOC_IN_RC(v18_veca_reg));
4980 match(vReg);
4981
4982 op_cost(0);
4983 format %{ %}
4984 interface(REG_INTER);
4985 %}
4986
4987 operand vReg_V23()
4988 %{
4989 constraint(ALLOC_IN_RC(v23_veca_reg));
4990 match(vReg);
4991
4992 op_cost(0);
4993 format %{ %}
4994 interface(REG_INTER);
4995 %}
4996
4997 operand vReg_V24()
4998 %{
4999 constraint(ALLOC_IN_RC(v24_veca_reg));
5000 match(vReg);
5001
5002 op_cost(0);
5003 format %{ %}
5004 interface(REG_INTER);
5005 %}
5006
5007 operand vecA()
5008 %{
5009 constraint(ALLOC_IN_RC(vectora_reg));
5010 match(VecA);
5011
5012 op_cost(0);
5013 format %{ %}
5014 interface(REG_INTER);
5015 %}
5016
5017 operand vecD()
5018 %{
5019 constraint(ALLOC_IN_RC(vectord_reg));
5020 match(VecD);
5021
5022 op_cost(0);
5023 format %{ %}
5024 interface(REG_INTER);
5025 %}
5026
5027 operand vecX()
5028 %{
5029 constraint(ALLOC_IN_RC(vectorx_reg));
5030 match(VecX);
5031
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand vRegD_V0()
5038 %{
5039 constraint(ALLOC_IN_RC(v0_reg));
5040 match(RegD);
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vRegD_V1()
5047 %{
5048 constraint(ALLOC_IN_RC(v1_reg));
5049 match(RegD);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 operand vRegD_V2()
5056 %{
5057 constraint(ALLOC_IN_RC(v2_reg));
5058 match(RegD);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vRegD_V3()
5065 %{
5066 constraint(ALLOC_IN_RC(v3_reg));
5067 match(RegD);
5068 op_cost(0);
5069 format %{ %}
5070 interface(REG_INTER);
5071 %}
5072
5073 operand vRegD_V4()
5074 %{
5075 constraint(ALLOC_IN_RC(v4_reg));
5076 match(RegD);
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vRegD_V5()
5083 %{
5084 constraint(ALLOC_IN_RC(v5_reg));
5085 match(RegD);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vRegD_V6()
5092 %{
5093 constraint(ALLOC_IN_RC(v6_reg));
5094 match(RegD);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vRegD_V7()
5101 %{
5102 constraint(ALLOC_IN_RC(v7_reg));
5103 match(RegD);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand vRegD_V12()
5110 %{
5111 constraint(ALLOC_IN_RC(v12_reg));
5112 match(RegD);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand vRegD_V13()
5119 %{
5120 constraint(ALLOC_IN_RC(v13_reg));
5121 match(RegD);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 operand pReg()
5128 %{
5129 constraint(ALLOC_IN_RC(pr_reg));
5130 match(RegVectMask);
5131 match(pRegGov);
5132 op_cost(0);
5133 format %{ %}
5134 interface(REG_INTER);
5135 %}
5136
5137 operand pRegGov()
5138 %{
5139 constraint(ALLOC_IN_RC(gov_pr));
5140 match(RegVectMask);
5141 match(pReg);
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 operand pRegGov_P0()
5148 %{
5149 constraint(ALLOC_IN_RC(p0_reg));
5150 match(RegVectMask);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 operand pRegGov_P1()
5157 %{
5158 constraint(ALLOC_IN_RC(p1_reg));
5159 match(RegVectMask);
5160 op_cost(0);
5161 format %{ %}
5162 interface(REG_INTER);
5163 %}
5164
5165 // Flags register, used as output of signed compare instructions
5166
5167 // note that on AArch64 we also use this register as the output for
5168 // for floating point compare instructions (CmpF CmpD). this ensures
5169 // that ordered inequality tests use GT, GE, LT or LE none of which
5170 // pass through cases where the result is unordered i.e. one or both
5171 // inputs to the compare is a NaN. this means that the ideal code can
5172 // replace e.g. a GT with an LE and not end up capturing the NaN case
5173 // (where the comparison should always fail). EQ and NE tests are
5174 // always generated in ideal code so that unordered folds into the NE
5175 // case, matching the behaviour of AArch64 NE.
5176 //
5177 // This differs from x86 where the outputs of FP compares use a
5178 // special FP flags registers and where compares based on this
5179 // register are distinguished into ordered inequalities (cmpOpUCF) and
5180 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5181 // to explicitly handle the unordered case in branches. x86 also has
5182 // to include extra CMoveX rules to accept a cmpOpUCF input.
5183
5184 operand rFlagsReg()
5185 %{
5186 constraint(ALLOC_IN_RC(int_flags));
5187 match(RegFlags);
5188
5189 op_cost(0);
5190 format %{ "RFLAGS" %}
5191 interface(REG_INTER);
5192 %}
5193
5194 // Flags register, used as output of unsigned compare instructions
5195 operand rFlagsRegU()
5196 %{
5197 constraint(ALLOC_IN_RC(int_flags));
5198 match(RegFlags);
5199
5200 op_cost(0);
5201 format %{ "RFLAGSU" %}
5202 interface(REG_INTER);
5203 %}
5204
5205 // Special Registers
5206
5207 // Method Register
5208 operand inline_cache_RegP(iRegP reg)
5209 %{
5210 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5211 match(reg);
5212 match(iRegPNoSp);
5213 op_cost(0);
5214 format %{ %}
5215 interface(REG_INTER);
5216 %}
5217
5218 // Thread Register
5219 operand thread_RegP(iRegP reg)
5220 %{
5221 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5222 match(reg);
5223 op_cost(0);
5224 format %{ %}
5225 interface(REG_INTER);
5226 %}
5227
5228 //----------Memory Operands----------------------------------------------------
5229
5230 operand indirect(iRegP reg)
5231 %{
5232 constraint(ALLOC_IN_RC(ptr_reg));
5233 match(reg);
5234 op_cost(0);
5235 format %{ "[$reg]" %}
5236 interface(MEMORY_INTER) %{
5237 base($reg);
5238 index(0xffffffff);
5239 scale(0x0);
5240 disp(0x0);
5241 %}
5242 %}
5243
5244 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5245 %{
5246 constraint(ALLOC_IN_RC(ptr_reg));
5247 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5248 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5249 op_cost(0);
5250 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5251 interface(MEMORY_INTER) %{
5252 base($reg);
5253 index($ireg);
5254 scale($scale);
5255 disp(0x0);
5256 %}
5257 %}
5258
5259 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5260 %{
5261 constraint(ALLOC_IN_RC(ptr_reg));
5262 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5263 match(AddP reg (LShiftL lreg scale));
5264 op_cost(0);
5265 format %{ "$reg, $lreg lsl($scale)" %}
5266 interface(MEMORY_INTER) %{
5267 base($reg);
5268 index($lreg);
5269 scale($scale);
5270 disp(0x0);
5271 %}
5272 %}
5273
5274 operand indIndexI2L(iRegP reg, iRegI ireg)
5275 %{
5276 constraint(ALLOC_IN_RC(ptr_reg));
5277 match(AddP reg (ConvI2L ireg));
5278 op_cost(0);
5279 format %{ "$reg, $ireg, 0, I2L" %}
5280 interface(MEMORY_INTER) %{
5281 base($reg);
5282 index($ireg);
5283 scale(0x0);
5284 disp(0x0);
5285 %}
5286 %}
5287
5288 operand indIndex(iRegP reg, iRegL lreg)
5289 %{
5290 constraint(ALLOC_IN_RC(ptr_reg));
5291 match(AddP reg lreg);
5292 op_cost(0);
5293 format %{ "$reg, $lreg" %}
5294 interface(MEMORY_INTER) %{
5295 base($reg);
5296 index($lreg);
5297 scale(0x0);
5298 disp(0x0);
5299 %}
5300 %}
5301
5302 operand indOffI1(iRegP reg, immIOffset1 off)
5303 %{
5304 constraint(ALLOC_IN_RC(ptr_reg));
5305 match(AddP reg off);
5306 op_cost(0);
5307 format %{ "[$reg, $off]" %}
5308 interface(MEMORY_INTER) %{
5309 base($reg);
5310 index(0xffffffff);
5311 scale(0x0);
5312 disp($off);
5313 %}
5314 %}
5315
5316 operand indOffI2(iRegP reg, immIOffset2 off)
5317 %{
5318 constraint(ALLOC_IN_RC(ptr_reg));
5319 match(AddP reg off);
5320 op_cost(0);
5321 format %{ "[$reg, $off]" %}
5322 interface(MEMORY_INTER) %{
5323 base($reg);
5324 index(0xffffffff);
5325 scale(0x0);
5326 disp($off);
5327 %}
5328 %}
5329
5330 operand indOffI4(iRegP reg, immIOffset4 off)
5331 %{
5332 constraint(ALLOC_IN_RC(ptr_reg));
5333 match(AddP reg off);
5334 op_cost(0);
5335 format %{ "[$reg, $off]" %}
5336 interface(MEMORY_INTER) %{
5337 base($reg);
5338 index(0xffffffff);
5339 scale(0x0);
5340 disp($off);
5341 %}
5342 %}
5343
5344 operand indOffI8(iRegP reg, immIOffset8 off)
5345 %{
5346 constraint(ALLOC_IN_RC(ptr_reg));
5347 match(AddP reg off);
5348 op_cost(0);
5349 format %{ "[$reg, $off]" %}
5350 interface(MEMORY_INTER) %{
5351 base($reg);
5352 index(0xffffffff);
5353 scale(0x0);
5354 disp($off);
5355 %}
5356 %}
5357
5358 operand indOffI16(iRegP reg, immIOffset16 off)
5359 %{
5360 constraint(ALLOC_IN_RC(ptr_reg));
5361 match(AddP reg off);
5362 op_cost(0);
5363 format %{ "[$reg, $off]" %}
5364 interface(MEMORY_INTER) %{
5365 base($reg);
5366 index(0xffffffff);
5367 scale(0x0);
5368 disp($off);
5369 %}
5370 %}
5371
5372 operand indOffL1(iRegP reg, immLoffset1 off)
5373 %{
5374 constraint(ALLOC_IN_RC(ptr_reg));
5375 match(AddP reg off);
5376 op_cost(0);
5377 format %{ "[$reg, $off]" %}
5378 interface(MEMORY_INTER) %{
5379 base($reg);
5380 index(0xffffffff);
5381 scale(0x0);
5382 disp($off);
5383 %}
5384 %}
5385
5386 operand indOffL2(iRegP reg, immLoffset2 off)
5387 %{
5388 constraint(ALLOC_IN_RC(ptr_reg));
5389 match(AddP reg off);
5390 op_cost(0);
5391 format %{ "[$reg, $off]" %}
5392 interface(MEMORY_INTER) %{
5393 base($reg);
5394 index(0xffffffff);
5395 scale(0x0);
5396 disp($off);
5397 %}
5398 %}
5399
5400 operand indOffL4(iRegP reg, immLoffset4 off)
5401 %{
5402 constraint(ALLOC_IN_RC(ptr_reg));
5403 match(AddP reg off);
5404 op_cost(0);
5405 format %{ "[$reg, $off]" %}
5406 interface(MEMORY_INTER) %{
5407 base($reg);
5408 index(0xffffffff);
5409 scale(0x0);
5410 disp($off);
5411 %}
5412 %}
5413
5414 operand indOffL8(iRegP reg, immLoffset8 off)
5415 %{
5416 constraint(ALLOC_IN_RC(ptr_reg));
5417 match(AddP reg off);
5418 op_cost(0);
5419 format %{ "[$reg, $off]" %}
5420 interface(MEMORY_INTER) %{
5421 base($reg);
5422 index(0xffffffff);
5423 scale(0x0);
5424 disp($off);
5425 %}
5426 %}
5427
5428 operand indOffL16(iRegP reg, immLoffset16 off)
5429 %{
5430 constraint(ALLOC_IN_RC(ptr_reg));
5431 match(AddP reg off);
5432 op_cost(0);
5433 format %{ "[$reg, $off]" %}
5434 interface(MEMORY_INTER) %{
5435 base($reg);
5436 index(0xffffffff);
5437 scale(0x0);
5438 disp($off);
5439 %}
5440 %}
5441
5442 operand indirectX2P(iRegL reg)
5443 %{
5444 constraint(ALLOC_IN_RC(ptr_reg));
5445 match(CastX2P reg);
5446 op_cost(0);
5447 format %{ "[$reg]\t# long -> ptr" %}
5448 interface(MEMORY_INTER) %{
5449 base($reg);
5450 index(0xffffffff);
5451 scale(0x0);
5452 disp(0x0);
5453 %}
5454 %}
5455
5456 operand indOffX2P(iRegL reg, immLOffset off)
5457 %{
5458 constraint(ALLOC_IN_RC(ptr_reg));
5459 match(AddP (CastX2P reg) off);
5460 op_cost(0);
5461 format %{ "[$reg, $off]\t# long -> ptr" %}
5462 interface(MEMORY_INTER) %{
5463 base($reg);
5464 index(0xffffffff);
5465 scale(0x0);
5466 disp($off);
5467 %}
5468 %}
5469
5470 operand indirectN(iRegN reg)
5471 %{
5472 predicate(CompressedOops::shift() == 0);
5473 constraint(ALLOC_IN_RC(ptr_reg));
5474 match(DecodeN reg);
5475 op_cost(0);
5476 format %{ "[$reg]\t# narrow" %}
5477 interface(MEMORY_INTER) %{
5478 base($reg);
5479 index(0xffffffff);
5480 scale(0x0);
5481 disp(0x0);
5482 %}
5483 %}
5484
5485 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5486 %{
5487 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5488 constraint(ALLOC_IN_RC(ptr_reg));
5489 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5490 op_cost(0);
5491 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5492 interface(MEMORY_INTER) %{
5493 base($reg);
5494 index($ireg);
5495 scale($scale);
5496 disp(0x0);
5497 %}
5498 %}
5499
5500 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5501 %{
5502 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(AddP (DecodeN reg) (LShiftL lreg scale));
5505 op_cost(0);
5506 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index($lreg);
5510 scale($scale);
5511 disp(0x0);
5512 %}
5513 %}
5514
5515 operand indIndexI2LN(iRegN reg, iRegI ireg)
5516 %{
5517 predicate(CompressedOops::shift() == 0);
5518 constraint(ALLOC_IN_RC(ptr_reg));
5519 match(AddP (DecodeN reg) (ConvI2L ireg));
5520 op_cost(0);
5521 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5522 interface(MEMORY_INTER) %{
5523 base($reg);
5524 index($ireg);
5525 scale(0x0);
5526 disp(0x0);
5527 %}
5528 %}
5529
5530 operand indIndexN(iRegN reg, iRegL lreg)
5531 %{
5532 predicate(CompressedOops::shift() == 0);
5533 constraint(ALLOC_IN_RC(ptr_reg));
5534 match(AddP (DecodeN reg) lreg);
5535 op_cost(0);
5536 format %{ "$reg, $lreg\t# narrow" %}
5537 interface(MEMORY_INTER) %{
5538 base($reg);
5539 index($lreg);
5540 scale(0x0);
5541 disp(0x0);
5542 %}
5543 %}
5544
5545 operand indOffIN(iRegN reg, immIOffset off)
5546 %{
5547 predicate(CompressedOops::shift() == 0);
5548 constraint(ALLOC_IN_RC(ptr_reg));
5549 match(AddP (DecodeN reg) off);
5550 op_cost(0);
5551 format %{ "[$reg, $off]\t# narrow" %}
5552 interface(MEMORY_INTER) %{
5553 base($reg);
5554 index(0xffffffff);
5555 scale(0x0);
5556 disp($off);
5557 %}
5558 %}
5559
5560 operand indOffLN(iRegN reg, immLOffset off)
5561 %{
5562 predicate(CompressedOops::shift() == 0);
5563 constraint(ALLOC_IN_RC(ptr_reg));
5564 match(AddP (DecodeN reg) off);
5565 op_cost(0);
5566 format %{ "[$reg, $off]\t# narrow" %}
5567 interface(MEMORY_INTER) %{
5568 base($reg);
5569 index(0xffffffff);
5570 scale(0x0);
5571 disp($off);
5572 %}
5573 %}
5574
5575
5576 //----------Special Memory Operands--------------------------------------------
5577 // Stack Slot Operand - This operand is used for loading and storing temporary
5578 // values on the stack where a match requires a value to
5579 // flow through memory.
5580 operand stackSlotP(sRegP reg)
5581 %{
5582 constraint(ALLOC_IN_RC(stack_slots));
5583 op_cost(100);
5584 // No match rule because this operand is only generated in matching
5585 // match(RegP);
5586 format %{ "[$reg]" %}
5587 interface(MEMORY_INTER) %{
5588 base(0x1e); // RSP
5589 index(0x0); // No Index
5590 scale(0x0); // No Scale
5591 disp($reg); // Stack Offset
5592 %}
5593 %}
5594
5595 operand stackSlotI(sRegI reg)
5596 %{
5597 constraint(ALLOC_IN_RC(stack_slots));
5598 // No match rule because this operand is only generated in matching
5599 // match(RegI);
5600 format %{ "[$reg]" %}
5601 interface(MEMORY_INTER) %{
5602 base(0x1e); // RSP
5603 index(0x0); // No Index
5604 scale(0x0); // No Scale
5605 disp($reg); // Stack Offset
5606 %}
5607 %}
5608
5609 operand stackSlotF(sRegF reg)
5610 %{
5611 constraint(ALLOC_IN_RC(stack_slots));
5612 // No match rule because this operand is only generated in matching
5613 // match(RegF);
5614 format %{ "[$reg]" %}
5615 interface(MEMORY_INTER) %{
5616 base(0x1e); // RSP
5617 index(0x0); // No Index
5618 scale(0x0); // No Scale
5619 disp($reg); // Stack Offset
5620 %}
5621 %}
5622
5623 operand stackSlotD(sRegD reg)
5624 %{
5625 constraint(ALLOC_IN_RC(stack_slots));
5626 // No match rule because this operand is only generated in matching
5627 // match(RegD);
5628 format %{ "[$reg]" %}
5629 interface(MEMORY_INTER) %{
5630 base(0x1e); // RSP
5631 index(0x0); // No Index
5632 scale(0x0); // No Scale
5633 disp($reg); // Stack Offset
5634 %}
5635 %}
5636
5637 operand stackSlotL(sRegL reg)
5638 %{
5639 constraint(ALLOC_IN_RC(stack_slots));
5640 // No match rule because this operand is only generated in matching
5641 // match(RegL);
5642 format %{ "[$reg]" %}
5643 interface(MEMORY_INTER) %{
5644 base(0x1e); // RSP
5645 index(0x0); // No Index
5646 scale(0x0); // No Scale
5647 disp($reg); // Stack Offset
5648 %}
5649 %}
5650
5651 // Operands for expressing Control Flow
5652 // NOTE: Label is a predefined operand which should not be redefined in
5653 // the AD file. It is generically handled within the ADLC.
5654
5655 //----------Conditional Branch Operands----------------------------------------
5656 // Comparison Op - This is the operation of the comparison, and is limited to
5657 // the following set of codes:
5658 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5659 //
5660 // Other attributes of the comparison, such as unsignedness, are specified
5661 // by the comparison instruction that sets a condition code flags register.
5662 // That result is represented by a flags operand whose subtype is appropriate
5663 // to the unsignedness (etc.) of the comparison.
5664 //
5665 // Later, the instruction which matches both the Comparison Op (a Bool) and
5666 // the flags (produced by the Cmp) specifies the coding of the comparison op
5667 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5668
5669 // used for signed integral comparisons and fp comparisons
5670
5671 operand cmpOp()
5672 %{
5673 match(Bool);
5674
5675 format %{ "" %}
5676 interface(COND_INTER) %{
5677 equal(0x0, "eq");
5678 not_equal(0x1, "ne");
5679 less(0xb, "lt");
5680 greater_equal(0xa, "ge");
5681 less_equal(0xd, "le");
5682 greater(0xc, "gt");
5683 overflow(0x6, "vs");
5684 no_overflow(0x7, "vc");
5685 %}
5686 %}
5687
5688 // used for unsigned integral comparisons
5689
5690 operand cmpOpU()
5691 %{
5692 match(Bool);
5693
5694 format %{ "" %}
5695 interface(COND_INTER) %{
5696 equal(0x0, "eq");
5697 not_equal(0x1, "ne");
5698 less(0x3, "lo");
5699 greater_equal(0x2, "hs");
5700 less_equal(0x9, "ls");
5701 greater(0x8, "hi");
5702 overflow(0x6, "vs");
5703 no_overflow(0x7, "vc");
5704 %}
5705 %}
5706
5707 // used for certain integral comparisons which can be
5708 // converted to cbxx or tbxx instructions
5709
5710 operand cmpOpEqNe()
5711 %{
5712 match(Bool);
5713 op_cost(0);
5714 predicate(n->as_Bool()->_test._test == BoolTest::ne
5715 || n->as_Bool()->_test._test == BoolTest::eq);
5716
5717 format %{ "" %}
5718 interface(COND_INTER) %{
5719 equal(0x0, "eq");
5720 not_equal(0x1, "ne");
5721 less(0xb, "lt");
5722 greater_equal(0xa, "ge");
5723 less_equal(0xd, "le");
5724 greater(0xc, "gt");
5725 overflow(0x6, "vs");
5726 no_overflow(0x7, "vc");
5727 %}
5728 %}
5729
5730 // used for certain integral comparisons which can be
5731 // converted to cbxx or tbxx instructions
5732
5733 operand cmpOpLtGe()
5734 %{
5735 match(Bool);
5736 op_cost(0);
5737
5738 predicate(n->as_Bool()->_test._test == BoolTest::lt
5739 || n->as_Bool()->_test._test == BoolTest::ge);
5740
5741 format %{ "" %}
5742 interface(COND_INTER) %{
5743 equal(0x0, "eq");
5744 not_equal(0x1, "ne");
5745 less(0xb, "lt");
5746 greater_equal(0xa, "ge");
5747 less_equal(0xd, "le");
5748 greater(0xc, "gt");
5749 overflow(0x6, "vs");
5750 no_overflow(0x7, "vc");
5751 %}
5752 %}
5753
5754 // used for certain unsigned integral comparisons which can be
5755 // converted to cbxx or tbxx instructions
5756
5757 operand cmpOpUEqNeLeGt()
5758 %{
5759 match(Bool);
5760 op_cost(0);
5761
5762 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5763 n->as_Bool()->_test._test == BoolTest::ne ||
5764 n->as_Bool()->_test._test == BoolTest::le ||
5765 n->as_Bool()->_test._test == BoolTest::gt);
5766
5767 format %{ "" %}
5768 interface(COND_INTER) %{
5769 equal(0x0, "eq");
5770 not_equal(0x1, "ne");
5771 less(0x3, "lo");
5772 greater_equal(0x2, "hs");
5773 less_equal(0x9, "ls");
5774 greater(0x8, "hi");
5775 overflow(0x6, "vs");
5776 no_overflow(0x7, "vc");
5777 %}
5778 %}
5779
5780 // Special operand allowing long args to int ops to be truncated for free
5781
5782 operand iRegL2I(iRegL reg) %{
5783
5784 op_cost(0);
5785
5786 match(ConvL2I reg);
5787
5788 format %{ "l2i($reg)" %}
5789
5790 interface(REG_INTER)
5791 %}
5792
5793 operand iRegL2P(iRegL reg) %{
5794
5795 op_cost(0);
5796
5797 match(CastX2P reg);
5798
5799 format %{ "l2p($reg)" %}
5800
5801 interface(REG_INTER)
5802 %}
5803
5804 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5805 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5806 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5807 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5808
5809 //----------OPERAND CLASSES----------------------------------------------------
5810 // Operand Classes are groups of operands that are used as to simplify
5811 // instruction definitions by not requiring the AD writer to specify
5812 // separate instructions for every form of operand when the
5813 // instruction accepts multiple operand types with the same basic
5814 // encoding and format. The classic case of this is memory operands.
5815
5816 // memory is used to define read/write location for load/store
5817 // instruction defs. we can turn a memory op into an Address
5818
5819 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5820 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5821
5822 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5823 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5824
5825 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5826 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5827
5828 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5829 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5830
5831 // All of the memory operands. For the pipeline description.
5832 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5833 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5834 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5835
5836
5837 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5838 // operations. it allows the src to be either an iRegI or a (ConvL2I
5839 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5840 // can be elided because the 32-bit instruction will just employ the
5841 // lower 32 bits anyway.
5842 //
5843 // n.b. this does not elide all L2I conversions. if the truncated
5844 // value is consumed by more than one operation then the ConvL2I
5845 // cannot be bundled into the consuming nodes so an l2i gets planted
5846 // (actually a movw $dst $src) and the downstream instructions consume
5847 // the result of the l2i as an iRegI input. That's a shame since the
5848 // movw is actually redundant but its not too costly.
5849
5850 opclass iRegIorL2I(iRegI, iRegL2I);
5851 opclass iRegPorL2P(iRegP, iRegL2P);
5852
5853 //----------PIPELINE-----------------------------------------------------------
5854 // Rules which define the behavior of the target architectures pipeline.
5855
5856 // For specific pipelines, eg A53, define the stages of that pipeline
5857 //pipe_desc(ISS, EX1, EX2, WR);
5858 #define ISS S0
5859 #define EX1 S1
5860 #define EX2 S2
5861 #define WR S3
5862
5863 // Integer ALU reg operation
5864 pipeline %{
5865
5866 attributes %{
5867 // ARM instructions are of fixed length
5868 fixed_size_instructions; // Fixed size instructions TODO does
5869 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5870 // ARM instructions come in 32-bit word units
5871 instruction_unit_size = 4; // An instruction is 4 bytes long
5872 instruction_fetch_unit_size = 64; // The processor fetches one line
5873 instruction_fetch_units = 1; // of 64 bytes
5874 %}
5875
5876 // We don't use an actual pipeline model so don't care about resources
5877 // or description. we do use pipeline classes to introduce fixed
5878 // latencies
5879
5880 //----------RESOURCES----------------------------------------------------------
5881 // Resources are the functional units available to the machine
5882
5883 resources( INS0, INS1, INS01 = INS0 | INS1,
5884 ALU0, ALU1, ALU = ALU0 | ALU1,
5885 MAC,
5886 DIV,
5887 BRANCH,
5888 LDST,
5889 NEON_FP);
5890
5891 //----------PIPELINE DESCRIPTION-----------------------------------------------
5892 // Pipeline Description specifies the stages in the machine's pipeline
5893
5894 // Define the pipeline as a generic 6 stage pipeline
5895 pipe_desc(S0, S1, S2, S3, S4, S5);
5896
5897 //----------PIPELINE CLASSES---------------------------------------------------
5898 // Pipeline Classes describe the stages in which input and output are
5899 // referenced by the hardware pipeline.
5900
5901 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5902 %{
5903 single_instruction;
5904 src1 : S1(read);
5905 src2 : S2(read);
5906 dst : S5(write);
5907 INS01 : ISS;
5908 NEON_FP : S5;
5909 %}
5910
5911 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5912 %{
5913 single_instruction;
5914 src1 : S1(read);
5915 src2 : S2(read);
5916 dst : S5(write);
5917 INS01 : ISS;
5918 NEON_FP : S5;
5919 %}
5920
5921 pipe_class fp_uop_s(vRegF dst, vRegF src)
5922 %{
5923 single_instruction;
5924 src : S1(read);
5925 dst : S5(write);
5926 INS01 : ISS;
5927 NEON_FP : S5;
5928 %}
5929
5930 pipe_class fp_uop_d(vRegD dst, vRegD src)
5931 %{
5932 single_instruction;
5933 src : S1(read);
5934 dst : S5(write);
5935 INS01 : ISS;
5936 NEON_FP : S5;
5937 %}
5938
5939 pipe_class fp_d2f(vRegF dst, vRegD src)
5940 %{
5941 single_instruction;
5942 src : S1(read);
5943 dst : S5(write);
5944 INS01 : ISS;
5945 NEON_FP : S5;
5946 %}
5947
5948 pipe_class fp_f2d(vRegD dst, vRegF src)
5949 %{
5950 single_instruction;
5951 src : S1(read);
5952 dst : S5(write);
5953 INS01 : ISS;
5954 NEON_FP : S5;
5955 %}
5956
5957 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5958 %{
5959 single_instruction;
5960 src : S1(read);
5961 dst : S5(write);
5962 INS01 : ISS;
5963 NEON_FP : S5;
5964 %}
5965
5966 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5967 %{
5968 single_instruction;
5969 src : S1(read);
5970 dst : S5(write);
5971 INS01 : ISS;
5972 NEON_FP : S5;
5973 %}
5974
5975 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5976 %{
5977 single_instruction;
5978 src : S1(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_l2f(vRegF dst, iRegL src)
5985 %{
5986 single_instruction;
5987 src : S1(read);
5988 dst : S5(write);
5989 INS01 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5994 %{
5995 single_instruction;
5996 src : S1(read);
5997 dst : S5(write);
5998 INS01 : ISS;
5999 NEON_FP : S5;
6000 %}
6001
6002 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6003 %{
6004 single_instruction;
6005 src : S1(read);
6006 dst : S5(write);
6007 INS01 : ISS;
6008 NEON_FP : S5;
6009 %}
6010
6011 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6012 %{
6013 single_instruction;
6014 src : S1(read);
6015 dst : S5(write);
6016 INS01 : ISS;
6017 NEON_FP : S5;
6018 %}
6019
6020 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6021 %{
6022 single_instruction;
6023 src : S1(read);
6024 dst : S5(write);
6025 INS01 : ISS;
6026 NEON_FP : S5;
6027 %}
6028
6029 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6030 %{
6031 single_instruction;
6032 src1 : S1(read);
6033 src2 : S2(read);
6034 dst : S5(write);
6035 INS0 : ISS;
6036 NEON_FP : S5;
6037 %}
6038
6039 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6040 %{
6041 single_instruction;
6042 src1 : S1(read);
6043 src2 : S2(read);
6044 dst : S5(write);
6045 INS0 : ISS;
6046 NEON_FP : S5;
6047 %}
6048
6049 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6050 %{
6051 single_instruction;
6052 cr : S1(read);
6053 src1 : S1(read);
6054 src2 : S1(read);
6055 dst : S3(write);
6056 INS01 : ISS;
6057 NEON_FP : S3;
6058 %}
6059
6060 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6061 %{
6062 single_instruction;
6063 cr : S1(read);
6064 src1 : S1(read);
6065 src2 : S1(read);
6066 dst : S3(write);
6067 INS01 : ISS;
6068 NEON_FP : S3;
6069 %}
6070
6071 pipe_class fp_imm_s(vRegF dst)
6072 %{
6073 single_instruction;
6074 dst : S3(write);
6075 INS01 : ISS;
6076 NEON_FP : S3;
6077 %}
6078
6079 pipe_class fp_imm_d(vRegD dst)
6080 %{
6081 single_instruction;
6082 dst : S3(write);
6083 INS01 : ISS;
6084 NEON_FP : S3;
6085 %}
6086
6087 pipe_class fp_load_constant_s(vRegF dst)
6088 %{
6089 single_instruction;
6090 dst : S4(write);
6091 INS01 : ISS;
6092 NEON_FP : S4;
6093 %}
6094
6095 pipe_class fp_load_constant_d(vRegD dst)
6096 %{
6097 single_instruction;
6098 dst : S4(write);
6099 INS01 : ISS;
6100 NEON_FP : S4;
6101 %}
6102
6103 //------- Integer ALU operations --------------------------
6104
6105 // Integer ALU reg-reg operation
6106 // Operands needed in EX1, result generated in EX2
6107 // Eg. ADD x0, x1, x2
6108 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6109 %{
6110 single_instruction;
6111 dst : EX2(write);
6112 src1 : EX1(read);
6113 src2 : EX1(read);
6114 INS01 : ISS; // Dual issue as instruction 0 or 1
6115 ALU : EX2;
6116 %}
6117
6118 // Integer ALU reg-reg operation with constant shift
6119 // Shifted register must be available in LATE_ISS instead of EX1
6120 // Eg. ADD x0, x1, x2, LSL #2
6121 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6122 %{
6123 single_instruction;
6124 dst : EX2(write);
6125 src1 : EX1(read);
6126 src2 : ISS(read);
6127 INS01 : ISS;
6128 ALU : EX2;
6129 %}
6130
6131 // Integer ALU reg operation with constant shift
6132 // Eg. LSL x0, x1, #shift
6133 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6134 %{
6135 single_instruction;
6136 dst : EX2(write);
6137 src1 : ISS(read);
6138 INS01 : ISS;
6139 ALU : EX2;
6140 %}
6141
6142 // Integer ALU reg-reg operation with variable shift
6143 // Both operands must be available in LATE_ISS instead of EX1
6144 // Result is available in EX1 instead of EX2
6145 // Eg. LSLV x0, x1, x2
6146 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6147 %{
6148 single_instruction;
6149 dst : EX1(write);
6150 src1 : ISS(read);
6151 src2 : ISS(read);
6152 INS01 : ISS;
6153 ALU : EX1;
6154 %}
6155
6156 // Integer ALU reg-reg operation with extract
6157 // As for _vshift above, but result generated in EX2
6158 // Eg. EXTR x0, x1, x2, #N
6159 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6160 %{
6161 single_instruction;
6162 dst : EX2(write);
6163 src1 : ISS(read);
6164 src2 : ISS(read);
6165 INS1 : ISS; // Can only dual issue as Instruction 1
6166 ALU : EX1;
6167 %}
6168
6169 // Integer ALU reg operation
6170 // Eg. NEG x0, x1
6171 pipe_class ialu_reg(iRegI dst, iRegI src)
6172 %{
6173 single_instruction;
6174 dst : EX2(write);
6175 src : EX1(read);
6176 INS01 : ISS;
6177 ALU : EX2;
6178 %}
6179
6180 // Integer ALU reg mmediate operation
6181 // Eg. ADD x0, x1, #N
6182 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6183 %{
6184 single_instruction;
6185 dst : EX2(write);
6186 src1 : EX1(read);
6187 INS01 : ISS;
6188 ALU : EX2;
6189 %}
6190
6191 // Integer ALU immediate operation (no source operands)
6192 // Eg. MOV x0, #N
6193 pipe_class ialu_imm(iRegI dst)
6194 %{
6195 single_instruction;
6196 dst : EX1(write);
6197 INS01 : ISS;
6198 ALU : EX1;
6199 %}
6200
6201 //------- Compare operation -------------------------------
6202
6203 // Compare reg-reg
6204 // Eg. CMP x0, x1
6205 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6206 %{
6207 single_instruction;
6208 // fixed_latency(16);
6209 cr : EX2(write);
6210 op1 : EX1(read);
6211 op2 : EX1(read);
6212 INS01 : ISS;
6213 ALU : EX2;
6214 %}
6215
6216 // Compare reg-reg
6217 // Eg. CMP x0, #N
6218 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6219 %{
6220 single_instruction;
6221 // fixed_latency(16);
6222 cr : EX2(write);
6223 op1 : EX1(read);
6224 INS01 : ISS;
6225 ALU : EX2;
6226 %}
6227
6228 //------- Conditional instructions ------------------------
6229
6230 // Conditional no operands
6231 // Eg. CSINC x0, zr, zr, <cond>
6232 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6233 %{
6234 single_instruction;
6235 cr : EX1(read);
6236 dst : EX2(write);
6237 INS01 : ISS;
6238 ALU : EX2;
6239 %}
6240
6241 // Conditional 2 operand
6242 // EG. CSEL X0, X1, X2, <cond>
6243 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6244 %{
6245 single_instruction;
6246 cr : EX1(read);
6247 src1 : EX1(read);
6248 src2 : EX1(read);
6249 dst : EX2(write);
6250 INS01 : ISS;
6251 ALU : EX2;
6252 %}
6253
6254 // Conditional 2 operand
6255 // EG. CSEL X0, X1, X2, <cond>
6256 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6257 %{
6258 single_instruction;
6259 cr : EX1(read);
6260 src : EX1(read);
6261 dst : EX2(write);
6262 INS01 : ISS;
6263 ALU : EX2;
6264 %}
6265
6266 //------- Multiply pipeline operations --------------------
6267
6268 // Multiply reg-reg
6269 // Eg. MUL w0, w1, w2
6270 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6271 %{
6272 single_instruction;
6273 dst : WR(write);
6274 src1 : ISS(read);
6275 src2 : ISS(read);
6276 INS01 : ISS;
6277 MAC : WR;
6278 %}
6279
6280 // Multiply accumulate
6281 // Eg. MADD w0, w1, w2, w3
6282 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6283 %{
6284 single_instruction;
6285 dst : WR(write);
6286 src1 : ISS(read);
6287 src2 : ISS(read);
6288 src3 : ISS(read);
6289 INS01 : ISS;
6290 MAC : WR;
6291 %}
6292
6293 // Eg. MUL w0, w1, w2
6294 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6295 %{
6296 single_instruction;
6297 fixed_latency(3); // Maximum latency for 64 bit mul
6298 dst : WR(write);
6299 src1 : ISS(read);
6300 src2 : ISS(read);
6301 INS01 : ISS;
6302 MAC : WR;
6303 %}
6304
6305 // Multiply accumulate
6306 // Eg. MADD w0, w1, w2, w3
6307 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6308 %{
6309 single_instruction;
6310 fixed_latency(3); // Maximum latency for 64 bit mul
6311 dst : WR(write);
6312 src1 : ISS(read);
6313 src2 : ISS(read);
6314 src3 : ISS(read);
6315 INS01 : ISS;
6316 MAC : WR;
6317 %}
6318
6319 //------- Divide pipeline operations --------------------
6320
6321 // Eg. SDIV w0, w1, w2
6322 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6323 %{
6324 single_instruction;
6325 fixed_latency(8); // Maximum latency for 32 bit divide
6326 dst : WR(write);
6327 src1 : ISS(read);
6328 src2 : ISS(read);
6329 INS0 : ISS; // Can only dual issue as instruction 0
6330 DIV : WR;
6331 %}
6332
6333 // Eg. SDIV x0, x1, x2
6334 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6335 %{
6336 single_instruction;
6337 fixed_latency(16); // Maximum latency for 64 bit divide
6338 dst : WR(write);
6339 src1 : ISS(read);
6340 src2 : ISS(read);
6341 INS0 : ISS; // Can only dual issue as instruction 0
6342 DIV : WR;
6343 %}
6344
6345 //------- Load pipeline operations ------------------------
6346
6347 // Load - prefetch
6348 // Eg. PFRM <mem>
6349 pipe_class iload_prefetch(memory mem)
6350 %{
6351 single_instruction;
6352 mem : ISS(read);
6353 INS01 : ISS;
6354 LDST : WR;
6355 %}
6356
6357 // Load - reg, mem
6358 // Eg. LDR x0, <mem>
6359 pipe_class iload_reg_mem(iRegI dst, memory mem)
6360 %{
6361 single_instruction;
6362 dst : WR(write);
6363 mem : ISS(read);
6364 INS01 : ISS;
6365 LDST : WR;
6366 %}
6367
6368 // Load - reg, reg
6369 // Eg. LDR x0, [sp, x1]
6370 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6371 %{
6372 single_instruction;
6373 dst : WR(write);
6374 src : ISS(read);
6375 INS01 : ISS;
6376 LDST : WR;
6377 %}
6378
6379 //------- Store pipeline operations -----------------------
6380
6381 // Store - zr, mem
6382 // Eg. STR zr, <mem>
6383 pipe_class istore_mem(memory mem)
6384 %{
6385 single_instruction;
6386 mem : ISS(read);
6387 INS01 : ISS;
6388 LDST : WR;
6389 %}
6390
6391 // Store - reg, mem
6392 // Eg. STR x0, <mem>
6393 pipe_class istore_reg_mem(iRegI src, memory mem)
6394 %{
6395 single_instruction;
6396 mem : ISS(read);
6397 src : EX2(read);
6398 INS01 : ISS;
6399 LDST : WR;
6400 %}
6401
6402 // Store - reg, reg
6403 // Eg. STR x0, [sp, x1]
6404 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6405 %{
6406 single_instruction;
6407 dst : ISS(read);
6408 src : EX2(read);
6409 INS01 : ISS;
6410 LDST : WR;
6411 %}
6412
6413 //------- Store pipeline operations -----------------------
6414
6415 // Branch
6416 pipe_class pipe_branch()
6417 %{
6418 single_instruction;
6419 INS01 : ISS;
6420 BRANCH : EX1;
6421 %}
6422
6423 // Conditional branch
6424 pipe_class pipe_branch_cond(rFlagsReg cr)
6425 %{
6426 single_instruction;
6427 cr : EX1(read);
6428 INS01 : ISS;
6429 BRANCH : EX1;
6430 %}
6431
6432 // Compare & Branch
6433 // EG. CBZ/CBNZ
6434 pipe_class pipe_cmp_branch(iRegI op1)
6435 %{
6436 single_instruction;
6437 op1 : EX1(read);
6438 INS01 : ISS;
6439 BRANCH : EX1;
6440 %}
6441
6442 //------- Synchronisation operations ----------------------
6443
6444 // Any operation requiring serialization.
6445 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6446 pipe_class pipe_serial()
6447 %{
6448 single_instruction;
6449 force_serialization;
6450 fixed_latency(16);
6451 INS01 : ISS(2); // Cannot dual issue with any other instruction
6452 LDST : WR;
6453 %}
6454
6455 // Generic big/slow expanded idiom - also serialized
6456 pipe_class pipe_slow()
6457 %{
6458 instruction_count(10);
6459 multiple_bundles;
6460 force_serialization;
6461 fixed_latency(16);
6462 INS01 : ISS(2); // Cannot dual issue with any other instruction
6463 LDST : WR;
6464 %}
6465
6466 // Empty pipeline class
6467 pipe_class pipe_class_empty()
6468 %{
6469 single_instruction;
6470 fixed_latency(0);
6471 %}
6472
6473 // Default pipeline class.
6474 pipe_class pipe_class_default()
6475 %{
6476 single_instruction;
6477 fixed_latency(2);
6478 %}
6479
6480 // Pipeline class for compares.
6481 pipe_class pipe_class_compare()
6482 %{
6483 single_instruction;
6484 fixed_latency(16);
6485 %}
6486
6487 // Pipeline class for memory operations.
6488 pipe_class pipe_class_memory()
6489 %{
6490 single_instruction;
6491 fixed_latency(16);
6492 %}
6493
6494 // Pipeline class for call.
6495 pipe_class pipe_class_call()
6496 %{
6497 single_instruction;
6498 fixed_latency(100);
6499 %}
6500
6501 // Define the class for the Nop node.
6502 define %{
6503 MachNop = pipe_class_empty;
6504 %}
6505
6506 %}
6507 //----------INSTRUCTIONS-------------------------------------------------------
6508 //
6509 // match -- States which machine-independent subtree may be replaced
6510 // by this instruction.
6511 // ins_cost -- The estimated cost of this instruction is used by instruction
6512 // selection to identify a minimum cost tree of machine
6513 // instructions that matches a tree of machine-independent
6514 // instructions.
6515 // format -- A string providing the disassembly for this instruction.
6516 // The value of an instruction's operand may be inserted
6517 // by referring to it with a '$' prefix.
6518 // opcode -- Three instruction opcodes may be provided. These are referred
6519 // to within an encode class as $primary, $secondary, and $tertiary
6520 // rrspectively. The primary opcode is commonly used to
6521 // indicate the type of machine instruction, while secondary
6522 // and tertiary are often used for prefix options or addressing
6523 // modes.
6524 // ins_encode -- A list of encode classes with parameters. The encode class
6525 // name must have been defined in an 'enc_class' specification
6526 // in the encode section of the architecture description.
6527
6528 // ============================================================================
6529 // Memory (Load/Store) Instructions
6530
6531 // Load Instructions
6532
6533 // Load Byte (8 bit signed)
6534 instruct loadB(iRegINoSp dst, memory1 mem)
6535 %{
6536 match(Set dst (LoadB mem));
6537 predicate(!needs_acquiring_load(n));
6538
6539 ins_cost(4 * INSN_COST);
6540 format %{ "ldrsbw $dst, $mem\t# byte" %}
6541
6542 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6543
6544 ins_pipe(iload_reg_mem);
6545 %}
6546
6547 // Load Byte (8 bit signed) into long
6548 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6549 %{
6550 match(Set dst (ConvI2L (LoadB mem)));
6551 predicate(!needs_acquiring_load(n->in(1)));
6552
6553 ins_cost(4 * INSN_COST);
6554 format %{ "ldrsb $dst, $mem\t# byte" %}
6555
6556 ins_encode(aarch64_enc_ldrsb(dst, mem));
6557
6558 ins_pipe(iload_reg_mem);
6559 %}
6560
6561 // Load Byte (8 bit unsigned)
6562 instruct loadUB(iRegINoSp dst, memory1 mem)
6563 %{
6564 match(Set dst (LoadUB mem));
6565 predicate(!needs_acquiring_load(n));
6566
6567 ins_cost(4 * INSN_COST);
6568 format %{ "ldrbw $dst, $mem\t# byte" %}
6569
6570 ins_encode(aarch64_enc_ldrb(dst, mem));
6571
6572 ins_pipe(iload_reg_mem);
6573 %}
6574
6575 // Load Byte (8 bit unsigned) into long
6576 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6577 %{
6578 match(Set dst (ConvI2L (LoadUB mem)));
6579 predicate(!needs_acquiring_load(n->in(1)));
6580
6581 ins_cost(4 * INSN_COST);
6582 format %{ "ldrb $dst, $mem\t# byte" %}
6583
6584 ins_encode(aarch64_enc_ldrb(dst, mem));
6585
6586 ins_pipe(iload_reg_mem);
6587 %}
6588
6589 // Load Short (16 bit signed)
6590 instruct loadS(iRegINoSp dst, memory2 mem)
6591 %{
6592 match(Set dst (LoadS mem));
6593 predicate(!needs_acquiring_load(n));
6594
6595 ins_cost(4 * INSN_COST);
6596 format %{ "ldrshw $dst, $mem\t# short" %}
6597
6598 ins_encode(aarch64_enc_ldrshw(dst, mem));
6599
6600 ins_pipe(iload_reg_mem);
6601 %}
6602
6603 // Load Short (16 bit signed) into long
6604 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6605 %{
6606 match(Set dst (ConvI2L (LoadS mem)));
6607 predicate(!needs_acquiring_load(n->in(1)));
6608
6609 ins_cost(4 * INSN_COST);
6610 format %{ "ldrsh $dst, $mem\t# short" %}
6611
6612 ins_encode(aarch64_enc_ldrsh(dst, mem));
6613
6614 ins_pipe(iload_reg_mem);
6615 %}
6616
6617 // Load Char (16 bit unsigned)
6618 instruct loadUS(iRegINoSp dst, memory2 mem)
6619 %{
6620 match(Set dst (LoadUS mem));
6621 predicate(!needs_acquiring_load(n));
6622
6623 ins_cost(4 * INSN_COST);
6624 format %{ "ldrh $dst, $mem\t# short" %}
6625
6626 ins_encode(aarch64_enc_ldrh(dst, mem));
6627
6628 ins_pipe(iload_reg_mem);
6629 %}
6630
6631 // Load Short/Char (16 bit unsigned) into long
6632 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6633 %{
6634 match(Set dst (ConvI2L (LoadUS mem)));
6635 predicate(!needs_acquiring_load(n->in(1)));
6636
6637 ins_cost(4 * INSN_COST);
6638 format %{ "ldrh $dst, $mem\t# short" %}
6639
6640 ins_encode(aarch64_enc_ldrh(dst, mem));
6641
6642 ins_pipe(iload_reg_mem);
6643 %}
6644
6645 // Load Integer (32 bit signed)
6646 instruct loadI(iRegINoSp dst, memory4 mem)
6647 %{
6648 match(Set dst (LoadI mem));
6649 predicate(!needs_acquiring_load(n));
6650
6651 ins_cost(4 * INSN_COST);
6652 format %{ "ldrw $dst, $mem\t# int" %}
6653
6654 ins_encode(aarch64_enc_ldrw(dst, mem));
6655
6656 ins_pipe(iload_reg_mem);
6657 %}
6658
6659 // Load Integer (32 bit signed) into long
6660 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6661 %{
6662 match(Set dst (ConvI2L (LoadI mem)));
6663 predicate(!needs_acquiring_load(n->in(1)));
6664
6665 ins_cost(4 * INSN_COST);
6666 format %{ "ldrsw $dst, $mem\t# int" %}
6667
6668 ins_encode(aarch64_enc_ldrsw(dst, mem));
6669
6670 ins_pipe(iload_reg_mem);
6671 %}
6672
6673 // Load Integer (32 bit unsigned) into long
6674 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6675 %{
6676 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6677 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6678
6679 ins_cost(4 * INSN_COST);
6680 format %{ "ldrw $dst, $mem\t# int" %}
6681
6682 ins_encode(aarch64_enc_ldrw(dst, mem));
6683
6684 ins_pipe(iload_reg_mem);
6685 %}
6686
6687 // Load Long (64 bit signed)
6688 instruct loadL(iRegLNoSp dst, memory8 mem)
6689 %{
6690 match(Set dst (LoadL mem));
6691 predicate(!needs_acquiring_load(n));
6692
6693 ins_cost(4 * INSN_COST);
6694 format %{ "ldr $dst, $mem\t# int" %}
6695
6696 ins_encode(aarch64_enc_ldr(dst, mem));
6697
6698 ins_pipe(iload_reg_mem);
6699 %}
6700
6701 // Load Range
6702 instruct loadRange(iRegINoSp dst, memory4 mem)
6703 %{
6704 match(Set dst (LoadRange mem));
6705
6706 ins_cost(4 * INSN_COST);
6707 format %{ "ldrw $dst, $mem\t# range" %}
6708
6709 ins_encode(aarch64_enc_ldrw(dst, mem));
6710
6711 ins_pipe(iload_reg_mem);
6712 %}
6713
6714 // Load Pointer
6715 instruct loadP(iRegPNoSp dst, memory8 mem)
6716 %{
6717 match(Set dst (LoadP mem));
6718 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6719
6720 ins_cost(4 * INSN_COST);
6721 format %{ "ldr $dst, $mem\t# ptr" %}
6722
6723 ins_encode(aarch64_enc_ldr(dst, mem));
6724
6725 ins_pipe(iload_reg_mem);
6726 %}
6727
6728 // Load Compressed Pointer
6729 instruct loadN(iRegNNoSp dst, memory4 mem)
6730 %{
6731 match(Set dst (LoadN mem));
6732 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6733
6734 ins_cost(4 * INSN_COST);
6735 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6736
6737 ins_encode(aarch64_enc_ldrw(dst, mem));
6738
6739 ins_pipe(iload_reg_mem);
6740 %}
6741
6742 // Load Klass Pointer
6743 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6744 %{
6745 match(Set dst (LoadKlass mem));
6746 predicate(!needs_acquiring_load(n));
6747
6748 ins_cost(4 * INSN_COST);
6749 format %{ "ldr $dst, $mem\t# class" %}
6750
6751 ins_encode(aarch64_enc_ldr(dst, mem));
6752
6753 ins_pipe(iload_reg_mem);
6754 %}
6755
6756 // Load Narrow Klass Pointer
6757 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6758 %{
6759 match(Set dst (LoadNKlass mem));
6760 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6761
6762 ins_cost(4 * INSN_COST);
6763 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6764
6765 ins_encode(aarch64_enc_ldrw(dst, mem));
6766
6767 ins_pipe(iload_reg_mem);
6768 %}
6769
6770 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6771 %{
6772 match(Set dst (LoadNKlass mem));
6773 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6774
6775 ins_cost(4 * INSN_COST);
6776 format %{
6777 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6778 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6779 %}
6780 ins_encode %{
6781 // inlined aarch64_enc_ldrw
6782 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6783 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6784 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6785 %}
6786 ins_pipe(iload_reg_mem);
6787 %}
6788
6789 // Load Float
6790 instruct loadF(vRegF dst, memory4 mem)
6791 %{
6792 match(Set dst (LoadF mem));
6793 predicate(!needs_acquiring_load(n));
6794
6795 ins_cost(4 * INSN_COST);
6796 format %{ "ldrs $dst, $mem\t# float" %}
6797
6798 ins_encode( aarch64_enc_ldrs(dst, mem) );
6799
6800 ins_pipe(pipe_class_memory);
6801 %}
6802
6803 // Load Double
6804 instruct loadD(vRegD dst, memory8 mem)
6805 %{
6806 match(Set dst (LoadD mem));
6807 predicate(!needs_acquiring_load(n));
6808
6809 ins_cost(4 * INSN_COST);
6810 format %{ "ldrd $dst, $mem\t# double" %}
6811
6812 ins_encode( aarch64_enc_ldrd(dst, mem) );
6813
6814 ins_pipe(pipe_class_memory);
6815 %}
6816
6817
6818 // Load Int Constant
6819 instruct loadConI(iRegINoSp dst, immI src)
6820 %{
6821 match(Set dst src);
6822
6823 ins_cost(INSN_COST);
6824 format %{ "mov $dst, $src\t# int" %}
6825
6826 ins_encode( aarch64_enc_movw_imm(dst, src) );
6827
6828 ins_pipe(ialu_imm);
6829 %}
6830
6831 // Load Long Constant
6832 instruct loadConL(iRegLNoSp dst, immL src)
6833 %{
6834 match(Set dst src);
6835
6836 ins_cost(INSN_COST);
6837 format %{ "mov $dst, $src\t# long" %}
6838
6839 ins_encode( aarch64_enc_mov_imm(dst, src) );
6840
6841 ins_pipe(ialu_imm);
6842 %}
6843
6844 // Load Pointer Constant
6845
6846 instruct loadConP(iRegPNoSp dst, immP con)
6847 %{
6848 match(Set dst con);
6849
6850 ins_cost(INSN_COST * 4);
6851 format %{
6852 "mov $dst, $con\t# ptr\n\t"
6853 %}
6854
6855 ins_encode(aarch64_enc_mov_p(dst, con));
6856
6857 ins_pipe(ialu_imm);
6858 %}
6859
6860 // Load Null Pointer Constant
6861
6862 instruct loadConP0(iRegPNoSp dst, immP0 con)
6863 %{
6864 match(Set dst con);
6865
6866 ins_cost(INSN_COST);
6867 format %{ "mov $dst, $con\t# nullptr ptr" %}
6868
6869 ins_encode(aarch64_enc_mov_p0(dst, con));
6870
6871 ins_pipe(ialu_imm);
6872 %}
6873
6874 // Load Pointer Constant One
6875
6876 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6877 %{
6878 match(Set dst con);
6879
6880 ins_cost(INSN_COST);
6881 format %{ "mov $dst, $con\t# nullptr ptr" %}
6882
6883 ins_encode(aarch64_enc_mov_p1(dst, con));
6884
6885 ins_pipe(ialu_imm);
6886 %}
6887
6888 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6889 %{
6890 match(Set dst con);
6891
6892 ins_cost(INSN_COST);
6893 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6894
6895 ins_encode %{
6896 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6897 %}
6898
6899 ins_pipe(ialu_imm);
6900 %}
6901
6902 // Load Narrow Pointer Constant
6903
6904 instruct loadConN(iRegNNoSp dst, immN con)
6905 %{
6906 match(Set dst con);
6907
6908 ins_cost(INSN_COST * 4);
6909 format %{ "mov $dst, $con\t# compressed ptr" %}
6910
6911 ins_encode(aarch64_enc_mov_n(dst, con));
6912
6913 ins_pipe(ialu_imm);
6914 %}
6915
6916 // Load Narrow Null Pointer Constant
6917
6918 instruct loadConN0(iRegNNoSp dst, immN0 con)
6919 %{
6920 match(Set dst con);
6921
6922 ins_cost(INSN_COST);
6923 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6924
6925 ins_encode(aarch64_enc_mov_n0(dst, con));
6926
6927 ins_pipe(ialu_imm);
6928 %}
6929
6930 // Load Narrow Klass Constant
6931
6932 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6933 %{
6934 match(Set dst con);
6935
6936 ins_cost(INSN_COST);
6937 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6938
6939 ins_encode(aarch64_enc_mov_nk(dst, con));
6940
6941 ins_pipe(ialu_imm);
6942 %}
6943
6944 // Load Packed Float Constant
6945
6946 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6947 match(Set dst con);
6948 ins_cost(INSN_COST * 4);
6949 format %{ "fmovs $dst, $con"%}
6950 ins_encode %{
6951 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6952 %}
6953
6954 ins_pipe(fp_imm_s);
6955 %}
6956
6957 // Load Float Constant
6958
6959 instruct loadConF(vRegF dst, immF con) %{
6960 match(Set dst con);
6961
6962 ins_cost(INSN_COST * 4);
6963
6964 format %{
6965 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6966 %}
6967
6968 ins_encode %{
6969 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6970 %}
6971
6972 ins_pipe(fp_load_constant_s);
6973 %}
6974
6975 // Load Packed Double Constant
6976
6977 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6978 match(Set dst con);
6979 ins_cost(INSN_COST);
6980 format %{ "fmovd $dst, $con"%}
6981 ins_encode %{
6982 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6983 %}
6984
6985 ins_pipe(fp_imm_d);
6986 %}
6987
6988 // Load Double Constant
6989
6990 instruct loadConD(vRegD dst, immD con) %{
6991 match(Set dst con);
6992
6993 ins_cost(INSN_COST * 5);
6994 format %{
6995 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6996 %}
6997
6998 ins_encode %{
6999 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7000 %}
7001
7002 ins_pipe(fp_load_constant_d);
7003 %}
7004
7005 // Load Half Float Constant
7006 instruct loadConH(vRegF dst, immH con) %{
7007 match(Set dst con);
7008 format %{ "mov rscratch1, $con\n\t"
7009 "fmov $dst, rscratch1"
7010 %}
7011 ins_encode %{
7012 __ movw(rscratch1, (uint32_t)$con$$constant);
7013 __ fmovs($dst$$FloatRegister, rscratch1);
7014 %}
7015 ins_pipe(pipe_class_default);
7016 %}
7017
7018 // Store Instructions
7019
7020 // Store Byte
7021 instruct storeB(iRegIorL2I src, memory1 mem)
7022 %{
7023 match(Set mem (StoreB mem src));
7024 predicate(!needs_releasing_store(n));
7025
7026 ins_cost(INSN_COST);
7027 format %{ "strb $src, $mem\t# byte" %}
7028
7029 ins_encode(aarch64_enc_strb(src, mem));
7030
7031 ins_pipe(istore_reg_mem);
7032 %}
7033
7034
7035 instruct storeimmB0(immI0 zero, memory1 mem)
7036 %{
7037 match(Set mem (StoreB mem zero));
7038 predicate(!needs_releasing_store(n));
7039
7040 ins_cost(INSN_COST);
7041 format %{ "strb rscractch2, $mem\t# byte" %}
7042
7043 ins_encode(aarch64_enc_strb0(mem));
7044
7045 ins_pipe(istore_mem);
7046 %}
7047
7048 // Store Char/Short
7049 instruct storeC(iRegIorL2I src, memory2 mem)
7050 %{
7051 match(Set mem (StoreC mem src));
7052 predicate(!needs_releasing_store(n));
7053
7054 ins_cost(INSN_COST);
7055 format %{ "strh $src, $mem\t# short" %}
7056
7057 ins_encode(aarch64_enc_strh(src, mem));
7058
7059 ins_pipe(istore_reg_mem);
7060 %}
7061
7062 instruct storeimmC0(immI0 zero, memory2 mem)
7063 %{
7064 match(Set mem (StoreC mem zero));
7065 predicate(!needs_releasing_store(n));
7066
7067 ins_cost(INSN_COST);
7068 format %{ "strh zr, $mem\t# short" %}
7069
7070 ins_encode(aarch64_enc_strh0(mem));
7071
7072 ins_pipe(istore_mem);
7073 %}
7074
7075 // Store Integer
7076
7077 instruct storeI(iRegIorL2I src, memory4 mem)
7078 %{
7079 match(Set mem(StoreI mem src));
7080 predicate(!needs_releasing_store(n));
7081
7082 ins_cost(INSN_COST);
7083 format %{ "strw $src, $mem\t# int" %}
7084
7085 ins_encode(aarch64_enc_strw(src, mem));
7086
7087 ins_pipe(istore_reg_mem);
7088 %}
7089
7090 instruct storeimmI0(immI0 zero, memory4 mem)
7091 %{
7092 match(Set mem(StoreI mem zero));
7093 predicate(!needs_releasing_store(n));
7094
7095 ins_cost(INSN_COST);
7096 format %{ "strw zr, $mem\t# int" %}
7097
7098 ins_encode(aarch64_enc_strw0(mem));
7099
7100 ins_pipe(istore_mem);
7101 %}
7102
7103 // Store Long (64 bit signed)
7104 instruct storeL(iRegL src, memory8 mem)
7105 %{
7106 match(Set mem (StoreL mem src));
7107 predicate(!needs_releasing_store(n));
7108
7109 ins_cost(INSN_COST);
7110 format %{ "str $src, $mem\t# int" %}
7111
7112 ins_encode(aarch64_enc_str(src, mem));
7113
7114 ins_pipe(istore_reg_mem);
7115 %}
7116
7117 // Store Long (64 bit signed)
7118 instruct storeimmL0(immL0 zero, memory8 mem)
7119 %{
7120 match(Set mem (StoreL mem zero));
7121 predicate(!needs_releasing_store(n));
7122
7123 ins_cost(INSN_COST);
7124 format %{ "str zr, $mem\t# int" %}
7125
7126 ins_encode(aarch64_enc_str0(mem));
7127
7128 ins_pipe(istore_mem);
7129 %}
7130
7131 // Store Pointer
7132 instruct storeP(iRegP src, memory8 mem)
7133 %{
7134 match(Set mem (StoreP mem src));
7135 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7136
7137 ins_cost(INSN_COST);
7138 format %{ "str $src, $mem\t# ptr" %}
7139
7140 ins_encode(aarch64_enc_str(src, mem));
7141
7142 ins_pipe(istore_reg_mem);
7143 %}
7144
7145 // Store Pointer
7146 instruct storeimmP0(immP0 zero, memory8 mem)
7147 %{
7148 match(Set mem (StoreP mem zero));
7149 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7150
7151 ins_cost(INSN_COST);
7152 format %{ "str zr, $mem\t# ptr" %}
7153
7154 ins_encode(aarch64_enc_str0(mem));
7155
7156 ins_pipe(istore_mem);
7157 %}
7158
7159 // Store Compressed Pointer
7160 instruct storeN(iRegN src, memory4 mem)
7161 %{
7162 match(Set mem (StoreN mem src));
7163 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7164
7165 ins_cost(INSN_COST);
7166 format %{ "strw $src, $mem\t# compressed ptr" %}
7167
7168 ins_encode(aarch64_enc_strw(src, mem));
7169
7170 ins_pipe(istore_reg_mem);
7171 %}
7172
7173 instruct storeImmN0(immN0 zero, memory4 mem)
7174 %{
7175 match(Set mem (StoreN mem zero));
7176 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7177
7178 ins_cost(INSN_COST);
7179 format %{ "strw zr, $mem\t# compressed ptr" %}
7180
7181 ins_encode(aarch64_enc_strw0(mem));
7182
7183 ins_pipe(istore_mem);
7184 %}
7185
7186 // Store Float
7187 instruct storeF(vRegF src, memory4 mem)
7188 %{
7189 match(Set mem (StoreF mem src));
7190 predicate(!needs_releasing_store(n));
7191
7192 ins_cost(INSN_COST);
7193 format %{ "strs $src, $mem\t# float" %}
7194
7195 ins_encode( aarch64_enc_strs(src, mem) );
7196
7197 ins_pipe(pipe_class_memory);
7198 %}
7199
7200 // TODO
7201 // implement storeImmF0 and storeFImmPacked
7202
7203 // Store Double
7204 instruct storeD(vRegD src, memory8 mem)
7205 %{
7206 match(Set mem (StoreD mem src));
7207 predicate(!needs_releasing_store(n));
7208
7209 ins_cost(INSN_COST);
7210 format %{ "strd $src, $mem\t# double" %}
7211
7212 ins_encode( aarch64_enc_strd(src, mem) );
7213
7214 ins_pipe(pipe_class_memory);
7215 %}
7216
7217 // Store Compressed Klass Pointer
7218 instruct storeNKlass(iRegN src, memory4 mem)
7219 %{
7220 predicate(!needs_releasing_store(n));
7221 match(Set mem (StoreNKlass mem src));
7222
7223 ins_cost(INSN_COST);
7224 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7225
7226 ins_encode(aarch64_enc_strw(src, mem));
7227
7228 ins_pipe(istore_reg_mem);
7229 %}
7230
7231 // TODO
7232 // implement storeImmD0 and storeDImmPacked
7233
7234 // prefetch instructions
7235 // Must be safe to execute with invalid address (cannot fault).
7236
7237 instruct prefetchalloc( memory8 mem ) %{
7238 match(PrefetchAllocation mem);
7239
7240 ins_cost(INSN_COST);
7241 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7242
7243 ins_encode( aarch64_enc_prefetchw(mem) );
7244
7245 ins_pipe(iload_prefetch);
7246 %}
7247
7248 // ---------------- volatile loads and stores ----------------
7249
7250 // Load Byte (8 bit signed)
7251 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7252 %{
7253 match(Set dst (LoadB mem));
7254
7255 ins_cost(VOLATILE_REF_COST);
7256 format %{ "ldarsb $dst, $mem\t# byte" %}
7257
7258 ins_encode(aarch64_enc_ldarsb(dst, mem));
7259
7260 ins_pipe(pipe_serial);
7261 %}
7262
7263 // Load Byte (8 bit signed) into long
7264 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7265 %{
7266 match(Set dst (ConvI2L (LoadB mem)));
7267
7268 ins_cost(VOLATILE_REF_COST);
7269 format %{ "ldarsb $dst, $mem\t# byte" %}
7270
7271 ins_encode(aarch64_enc_ldarsb(dst, mem));
7272
7273 ins_pipe(pipe_serial);
7274 %}
7275
7276 // Load Byte (8 bit unsigned)
7277 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7278 %{
7279 match(Set dst (LoadUB mem));
7280
7281 ins_cost(VOLATILE_REF_COST);
7282 format %{ "ldarb $dst, $mem\t# byte" %}
7283
7284 ins_encode(aarch64_enc_ldarb(dst, mem));
7285
7286 ins_pipe(pipe_serial);
7287 %}
7288
7289 // Load Byte (8 bit unsigned) into long
7290 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7291 %{
7292 match(Set dst (ConvI2L (LoadUB mem)));
7293
7294 ins_cost(VOLATILE_REF_COST);
7295 format %{ "ldarb $dst, $mem\t# byte" %}
7296
7297 ins_encode(aarch64_enc_ldarb(dst, mem));
7298
7299 ins_pipe(pipe_serial);
7300 %}
7301
7302 // Load Short (16 bit signed)
7303 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7304 %{
7305 match(Set dst (LoadS mem));
7306
7307 ins_cost(VOLATILE_REF_COST);
7308 format %{ "ldarshw $dst, $mem\t# short" %}
7309
7310 ins_encode(aarch64_enc_ldarshw(dst, mem));
7311
7312 ins_pipe(pipe_serial);
7313 %}
7314
7315 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7316 %{
7317 match(Set dst (LoadUS mem));
7318
7319 ins_cost(VOLATILE_REF_COST);
7320 format %{ "ldarhw $dst, $mem\t# short" %}
7321
7322 ins_encode(aarch64_enc_ldarhw(dst, mem));
7323
7324 ins_pipe(pipe_serial);
7325 %}
7326
7327 // Load Short/Char (16 bit unsigned) into long
7328 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7329 %{
7330 match(Set dst (ConvI2L (LoadUS mem)));
7331
7332 ins_cost(VOLATILE_REF_COST);
7333 format %{ "ldarh $dst, $mem\t# short" %}
7334
7335 ins_encode(aarch64_enc_ldarh(dst, mem));
7336
7337 ins_pipe(pipe_serial);
7338 %}
7339
7340 // Load Short/Char (16 bit signed) into long
7341 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7342 %{
7343 match(Set dst (ConvI2L (LoadS mem)));
7344
7345 ins_cost(VOLATILE_REF_COST);
7346 format %{ "ldarh $dst, $mem\t# short" %}
7347
7348 ins_encode(aarch64_enc_ldarsh(dst, mem));
7349
7350 ins_pipe(pipe_serial);
7351 %}
7352
7353 // Load Integer (32 bit signed)
7354 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7355 %{
7356 match(Set dst (LoadI mem));
7357
7358 ins_cost(VOLATILE_REF_COST);
7359 format %{ "ldarw $dst, $mem\t# int" %}
7360
7361 ins_encode(aarch64_enc_ldarw(dst, mem));
7362
7363 ins_pipe(pipe_serial);
7364 %}
7365
7366 // Load Integer (32 bit unsigned) into long
7367 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7368 %{
7369 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7370
7371 ins_cost(VOLATILE_REF_COST);
7372 format %{ "ldarw $dst, $mem\t# int" %}
7373
7374 ins_encode(aarch64_enc_ldarw(dst, mem));
7375
7376 ins_pipe(pipe_serial);
7377 %}
7378
7379 // Load Long (64 bit signed)
7380 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7381 %{
7382 match(Set dst (LoadL mem));
7383
7384 ins_cost(VOLATILE_REF_COST);
7385 format %{ "ldar $dst, $mem\t# int" %}
7386
7387 ins_encode(aarch64_enc_ldar(dst, mem));
7388
7389 ins_pipe(pipe_serial);
7390 %}
7391
7392 // Load Pointer
7393 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7394 %{
7395 match(Set dst (LoadP mem));
7396 predicate(n->as_Load()->barrier_data() == 0);
7397
7398 ins_cost(VOLATILE_REF_COST);
7399 format %{ "ldar $dst, $mem\t# ptr" %}
7400
7401 ins_encode(aarch64_enc_ldar(dst, mem));
7402
7403 ins_pipe(pipe_serial);
7404 %}
7405
7406 // Load Compressed Pointer
7407 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7408 %{
7409 match(Set dst (LoadN mem));
7410 predicate(n->as_Load()->barrier_data() == 0);
7411
7412 ins_cost(VOLATILE_REF_COST);
7413 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7414
7415 ins_encode(aarch64_enc_ldarw(dst, mem));
7416
7417 ins_pipe(pipe_serial);
7418 %}
7419
7420 // Load Float
7421 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7422 %{
7423 match(Set dst (LoadF mem));
7424
7425 ins_cost(VOLATILE_REF_COST);
7426 format %{ "ldars $dst, $mem\t# float" %}
7427
7428 ins_encode( aarch64_enc_fldars(dst, mem) );
7429
7430 ins_pipe(pipe_serial);
7431 %}
7432
7433 // Load Double
7434 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7435 %{
7436 match(Set dst (LoadD mem));
7437
7438 ins_cost(VOLATILE_REF_COST);
7439 format %{ "ldard $dst, $mem\t# double" %}
7440
7441 ins_encode( aarch64_enc_fldard(dst, mem) );
7442
7443 ins_pipe(pipe_serial);
7444 %}
7445
7446 // Store Byte
7447 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7448 %{
7449 match(Set mem (StoreB mem src));
7450
7451 ins_cost(VOLATILE_REF_COST);
7452 format %{ "stlrb $src, $mem\t# byte" %}
7453
7454 ins_encode(aarch64_enc_stlrb(src, mem));
7455
7456 ins_pipe(pipe_class_memory);
7457 %}
7458
7459 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7460 %{
7461 match(Set mem (StoreB mem zero));
7462
7463 ins_cost(VOLATILE_REF_COST);
7464 format %{ "stlrb zr, $mem\t# byte" %}
7465
7466 ins_encode(aarch64_enc_stlrb0(mem));
7467
7468 ins_pipe(pipe_class_memory);
7469 %}
7470
7471 // Store Char/Short
7472 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7473 %{
7474 match(Set mem (StoreC mem src));
7475
7476 ins_cost(VOLATILE_REF_COST);
7477 format %{ "stlrh $src, $mem\t# short" %}
7478
7479 ins_encode(aarch64_enc_stlrh(src, mem));
7480
7481 ins_pipe(pipe_class_memory);
7482 %}
7483
7484 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7485 %{
7486 match(Set mem (StoreC mem zero));
7487
7488 ins_cost(VOLATILE_REF_COST);
7489 format %{ "stlrh zr, $mem\t# short" %}
7490
7491 ins_encode(aarch64_enc_stlrh0(mem));
7492
7493 ins_pipe(pipe_class_memory);
7494 %}
7495
7496 // Store Integer
7497
7498 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7499 %{
7500 match(Set mem(StoreI mem src));
7501
7502 ins_cost(VOLATILE_REF_COST);
7503 format %{ "stlrw $src, $mem\t# int" %}
7504
7505 ins_encode(aarch64_enc_stlrw(src, mem));
7506
7507 ins_pipe(pipe_class_memory);
7508 %}
7509
7510 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7511 %{
7512 match(Set mem(StoreI mem zero));
7513
7514 ins_cost(VOLATILE_REF_COST);
7515 format %{ "stlrw zr, $mem\t# int" %}
7516
7517 ins_encode(aarch64_enc_stlrw0(mem));
7518
7519 ins_pipe(pipe_class_memory);
7520 %}
7521
7522 // Store Long (64 bit signed)
7523 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7524 %{
7525 match(Set mem (StoreL mem src));
7526
7527 ins_cost(VOLATILE_REF_COST);
7528 format %{ "stlr $src, $mem\t# int" %}
7529
7530 ins_encode(aarch64_enc_stlr(src, mem));
7531
7532 ins_pipe(pipe_class_memory);
7533 %}
7534
7535 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7536 %{
7537 match(Set mem (StoreL mem zero));
7538
7539 ins_cost(VOLATILE_REF_COST);
7540 format %{ "stlr zr, $mem\t# int" %}
7541
7542 ins_encode(aarch64_enc_stlr0(mem));
7543
7544 ins_pipe(pipe_class_memory);
7545 %}
7546
7547 // Store Pointer
7548 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7549 %{
7550 match(Set mem (StoreP mem src));
7551 predicate(n->as_Store()->barrier_data() == 0);
7552
7553 ins_cost(VOLATILE_REF_COST);
7554 format %{ "stlr $src, $mem\t# ptr" %}
7555
7556 ins_encode(aarch64_enc_stlr(src, mem));
7557
7558 ins_pipe(pipe_class_memory);
7559 %}
7560
7561 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7562 %{
7563 match(Set mem (StoreP mem zero));
7564 predicate(n->as_Store()->barrier_data() == 0);
7565
7566 ins_cost(VOLATILE_REF_COST);
7567 format %{ "stlr zr, $mem\t# ptr" %}
7568
7569 ins_encode(aarch64_enc_stlr0(mem));
7570
7571 ins_pipe(pipe_class_memory);
7572 %}
7573
7574 // Store Compressed Pointer
7575 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7576 %{
7577 match(Set mem (StoreN mem src));
7578 predicate(n->as_Store()->barrier_data() == 0);
7579
7580 ins_cost(VOLATILE_REF_COST);
7581 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7582
7583 ins_encode(aarch64_enc_stlrw(src, mem));
7584
7585 ins_pipe(pipe_class_memory);
7586 %}
7587
7588 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7589 %{
7590 match(Set mem (StoreN mem zero));
7591 predicate(n->as_Store()->barrier_data() == 0);
7592
7593 ins_cost(VOLATILE_REF_COST);
7594 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7595
7596 ins_encode(aarch64_enc_stlrw0(mem));
7597
7598 ins_pipe(pipe_class_memory);
7599 %}
7600
7601 // Store Float
7602 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7603 %{
7604 match(Set mem (StoreF mem src));
7605
7606 ins_cost(VOLATILE_REF_COST);
7607 format %{ "stlrs $src, $mem\t# float" %}
7608
7609 ins_encode( aarch64_enc_fstlrs(src, mem) );
7610
7611 ins_pipe(pipe_class_memory);
7612 %}
7613
7614 // TODO
7615 // implement storeImmF0 and storeFImmPacked
7616
7617 // Store Double
7618 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7619 %{
7620 match(Set mem (StoreD mem src));
7621
7622 ins_cost(VOLATILE_REF_COST);
7623 format %{ "stlrd $src, $mem\t# double" %}
7624
7625 ins_encode( aarch64_enc_fstlrd(src, mem) );
7626
7627 ins_pipe(pipe_class_memory);
7628 %}
7629
7630 // ---------------- end of volatile loads and stores ----------------
7631
7632 instruct cacheWB(indirect addr)
7633 %{
7634 predicate(VM_Version::supports_data_cache_line_flush());
7635 match(CacheWB addr);
7636
7637 ins_cost(100);
7638 format %{"cache wb $addr" %}
7639 ins_encode %{
7640 assert($addr->index_position() < 0, "should be");
7641 assert($addr$$disp == 0, "should be");
7642 __ cache_wb(Address($addr$$base$$Register, 0));
7643 %}
7644 ins_pipe(pipe_slow); // XXX
7645 %}
7646
7647 instruct cacheWBPreSync()
7648 %{
7649 predicate(VM_Version::supports_data_cache_line_flush());
7650 match(CacheWBPreSync);
7651
7652 ins_cost(100);
7653 format %{"cache wb presync" %}
7654 ins_encode %{
7655 __ cache_wbsync(true);
7656 %}
7657 ins_pipe(pipe_slow); // XXX
7658 %}
7659
7660 instruct cacheWBPostSync()
7661 %{
7662 predicate(VM_Version::supports_data_cache_line_flush());
7663 match(CacheWBPostSync);
7664
7665 ins_cost(100);
7666 format %{"cache wb postsync" %}
7667 ins_encode %{
7668 __ cache_wbsync(false);
7669 %}
7670 ins_pipe(pipe_slow); // XXX
7671 %}
7672
7673 // ============================================================================
7674 // BSWAP Instructions
7675
7676 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7677 match(Set dst (ReverseBytesI src));
7678
7679 ins_cost(INSN_COST);
7680 format %{ "revw $dst, $src" %}
7681
7682 ins_encode %{
7683 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7684 %}
7685
7686 ins_pipe(ialu_reg);
7687 %}
7688
7689 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7690 match(Set dst (ReverseBytesL src));
7691
7692 ins_cost(INSN_COST);
7693 format %{ "rev $dst, $src" %}
7694
7695 ins_encode %{
7696 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7697 %}
7698
7699 ins_pipe(ialu_reg);
7700 %}
7701
7702 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7703 match(Set dst (ReverseBytesUS src));
7704
7705 ins_cost(INSN_COST);
7706 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7707
7708 ins_encode %{
7709 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7710 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7711 %}
7712
7713 ins_pipe(ialu_reg);
7714 %}
7715
7716 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7717 match(Set dst (ReverseBytesS src));
7718
7719 ins_cost(INSN_COST);
7720 format %{ "rev16w $dst, $src\n\t"
7721 "sbfmw $dst, $dst, #0, #15" %}
7722
7723 ins_encode %{
7724 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7725 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7726 %}
7727
7728 ins_pipe(ialu_reg);
7729 %}
7730
7731 // ============================================================================
7732 // Zero Count Instructions
7733
7734 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7735 match(Set dst (CountLeadingZerosI src));
7736
7737 ins_cost(INSN_COST);
7738 format %{ "clzw $dst, $src" %}
7739 ins_encode %{
7740 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7741 %}
7742
7743 ins_pipe(ialu_reg);
7744 %}
7745
7746 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7747 match(Set dst (CountLeadingZerosL src));
7748
7749 ins_cost(INSN_COST);
7750 format %{ "clz $dst, $src" %}
7751 ins_encode %{
7752 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7753 %}
7754
7755 ins_pipe(ialu_reg);
7756 %}
7757
7758 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7759 match(Set dst (CountTrailingZerosI src));
7760
7761 ins_cost(INSN_COST * 2);
7762 format %{ "rbitw $dst, $src\n\t"
7763 "clzw $dst, $dst" %}
7764 ins_encode %{
7765 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7766 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7767 %}
7768
7769 ins_pipe(ialu_reg);
7770 %}
7771
7772 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7773 match(Set dst (CountTrailingZerosL src));
7774
7775 ins_cost(INSN_COST * 2);
7776 format %{ "rbit $dst, $src\n\t"
7777 "clz $dst, $dst" %}
7778 ins_encode %{
7779 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7780 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7781 %}
7782
7783 ins_pipe(ialu_reg);
7784 %}
7785
7786 //---------- Population Count Instructions -------------------------------------
7787 //
7788
7789 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7790 match(Set dst (PopCountI src));
7791 effect(TEMP tmp);
7792 ins_cost(INSN_COST * 13);
7793
7794 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7795 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7796 "addv $tmp, $tmp\t# vector (8B)\n\t"
7797 "mov $dst, $tmp\t# vector (1D)" %}
7798 ins_encode %{
7799 __ fmovs($tmp$$FloatRegister, $src$$Register);
7800 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7801 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7802 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7803 %}
7804
7805 ins_pipe(pipe_class_default);
7806 %}
7807
7808 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7809 match(Set dst (PopCountI (LoadI mem)));
7810 effect(TEMP tmp);
7811 ins_cost(INSN_COST * 13);
7812
7813 format %{ "ldrs $tmp, $mem\n\t"
7814 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7815 "addv $tmp, $tmp\t# vector (8B)\n\t"
7816 "mov $dst, $tmp\t# vector (1D)" %}
7817 ins_encode %{
7818 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7819 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7820 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7821 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7822 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7823 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7824 %}
7825
7826 ins_pipe(pipe_class_default);
7827 %}
7828
7829 // Note: Long.bitCount(long) returns an int.
7830 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7831 match(Set dst (PopCountL src));
7832 effect(TEMP tmp);
7833 ins_cost(INSN_COST * 13);
7834
7835 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7836 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7837 "addv $tmp, $tmp\t# vector (8B)\n\t"
7838 "mov $dst, $tmp\t# vector (1D)" %}
7839 ins_encode %{
7840 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7841 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7842 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7843 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7844 %}
7845
7846 ins_pipe(pipe_class_default);
7847 %}
7848
7849 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7850 match(Set dst (PopCountL (LoadL mem)));
7851 effect(TEMP tmp);
7852 ins_cost(INSN_COST * 13);
7853
7854 format %{ "ldrd $tmp, $mem\n\t"
7855 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7856 "addv $tmp, $tmp\t# vector (8B)\n\t"
7857 "mov $dst, $tmp\t# vector (1D)" %}
7858 ins_encode %{
7859 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7860 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7861 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7862 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7863 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7864 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7865 %}
7866
7867 ins_pipe(pipe_class_default);
7868 %}
7869
7870 // ============================================================================
7871 // VerifyVectorAlignment Instruction
7872
7873 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7874 match(Set addr (VerifyVectorAlignment addr mask));
7875 effect(KILL cr);
7876 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7877 ins_encode %{
7878 Label Lskip;
7879 // check if masked bits of addr are zero
7880 __ tst($addr$$Register, $mask$$constant);
7881 __ br(Assembler::EQ, Lskip);
7882 __ stop("verify_vector_alignment found a misaligned vector memory access");
7883 __ bind(Lskip);
7884 %}
7885 ins_pipe(pipe_slow);
7886 %}
7887
7888 // ============================================================================
7889 // MemBar Instruction
7890
7891 instruct load_fence() %{
7892 match(LoadFence);
7893 ins_cost(VOLATILE_REF_COST);
7894
7895 format %{ "load_fence" %}
7896
7897 ins_encode %{
7898 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7899 %}
7900 ins_pipe(pipe_serial);
7901 %}
7902
7903 instruct unnecessary_membar_acquire() %{
7904 predicate(unnecessary_acquire(n));
7905 match(MemBarAcquire);
7906 ins_cost(0);
7907
7908 format %{ "membar_acquire (elided)" %}
7909
7910 ins_encode %{
7911 __ block_comment("membar_acquire (elided)");
7912 %}
7913
7914 ins_pipe(pipe_class_empty);
7915 %}
7916
7917 instruct membar_acquire() %{
7918 match(MemBarAcquire);
7919 ins_cost(VOLATILE_REF_COST);
7920
7921 format %{ "membar_acquire\n\t"
7922 "dmb ishld" %}
7923
7924 ins_encode %{
7925 __ block_comment("membar_acquire");
7926 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7927 %}
7928
7929 ins_pipe(pipe_serial);
7930 %}
7931
7932
7933 instruct membar_acquire_lock() %{
7934 match(MemBarAcquireLock);
7935 ins_cost(VOLATILE_REF_COST);
7936
7937 format %{ "membar_acquire_lock (elided)" %}
7938
7939 ins_encode %{
7940 __ block_comment("membar_acquire_lock (elided)");
7941 %}
7942
7943 ins_pipe(pipe_serial);
7944 %}
7945
7946 instruct store_fence() %{
7947 match(StoreFence);
7948 ins_cost(VOLATILE_REF_COST);
7949
7950 format %{ "store_fence" %}
7951
7952 ins_encode %{
7953 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7954 %}
7955 ins_pipe(pipe_serial);
7956 %}
7957
7958 instruct unnecessary_membar_release() %{
7959 predicate(unnecessary_release(n));
7960 match(MemBarRelease);
7961 ins_cost(0);
7962
7963 format %{ "membar_release (elided)" %}
7964
7965 ins_encode %{
7966 __ block_comment("membar_release (elided)");
7967 %}
7968 ins_pipe(pipe_serial);
7969 %}
7970
7971 instruct membar_release() %{
7972 match(MemBarRelease);
7973 ins_cost(VOLATILE_REF_COST);
7974
7975 format %{ "membar_release\n\t"
7976 "dmb ishst\n\tdmb ishld" %}
7977
7978 ins_encode %{
7979 __ block_comment("membar_release");
7980 // These will be merged if AlwaysMergeDMB is enabled.
7981 __ membar(Assembler::StoreStore);
7982 __ membar(Assembler::LoadStore);
7983 %}
7984 ins_pipe(pipe_serial);
7985 %}
7986
7987 instruct membar_storestore() %{
7988 match(MemBarStoreStore);
7989 match(StoreStoreFence);
7990 ins_cost(VOLATILE_REF_COST);
7991
7992 format %{ "MEMBAR-store-store" %}
7993
7994 ins_encode %{
7995 __ membar(Assembler::StoreStore);
7996 %}
7997 ins_pipe(pipe_serial);
7998 %}
7999
8000 instruct membar_release_lock() %{
8001 match(MemBarReleaseLock);
8002 ins_cost(VOLATILE_REF_COST);
8003
8004 format %{ "membar_release_lock (elided)" %}
8005
8006 ins_encode %{
8007 __ block_comment("membar_release_lock (elided)");
8008 %}
8009
8010 ins_pipe(pipe_serial);
8011 %}
8012
8013 instruct membar_storeload() %{
8014 match(MemBarStoreLoad);
8015 ins_cost(VOLATILE_REF_COST*100);
8016
8017 format %{ "MEMBAR-store-load\n\t"
8018 "dmb ish" %}
8019
8020 ins_encode %{
8021 __ block_comment("membar_storeload");
8022 __ membar(Assembler::StoreLoad);
8023 %}
8024
8025 ins_pipe(pipe_serial);
8026 %}
8027
8028 instruct unnecessary_membar_volatile() %{
8029 predicate(unnecessary_volatile(n));
8030 match(MemBarVolatile);
8031 ins_cost(0);
8032
8033 format %{ "membar_volatile (elided)" %}
8034
8035 ins_encode %{
8036 __ block_comment("membar_volatile (elided)");
8037 %}
8038
8039 ins_pipe(pipe_serial);
8040 %}
8041
8042 instruct membar_volatile() %{
8043 match(MemBarVolatile);
8044 ins_cost(VOLATILE_REF_COST*100);
8045
8046 format %{ "membar_volatile\n\t"
8047 "dmb ish"%}
8048
8049 ins_encode %{
8050 __ block_comment("membar_volatile");
8051 __ membar(Assembler::StoreLoad);
8052 %}
8053
8054 ins_pipe(pipe_serial);
8055 %}
8056
8057 instruct membar_full() %{
8058 match(MemBarFull);
8059 ins_cost(VOLATILE_REF_COST*100);
8060
8061 format %{ "membar_full\n\t"
8062 "dmb ish" %}
8063 ins_encode %{
8064 __ block_comment("membar_full");
8065 __ membar(Assembler::AnyAny);
8066 %}
8067
8068 ins_pipe(pipe_serial);
8069 %}
8070
8071 // ============================================================================
8072 // Cast/Convert Instructions
8073
8074 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8075 match(Set dst (CastX2P src));
8076
8077 ins_cost(INSN_COST);
8078 format %{ "mov $dst, $src\t# long -> ptr" %}
8079
8080 ins_encode %{
8081 if ($dst$$reg != $src$$reg) {
8082 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8083 }
8084 %}
8085
8086 ins_pipe(ialu_reg);
8087 %}
8088
8089 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8090 match(Set dst (CastP2X src));
8091
8092 ins_cost(INSN_COST);
8093 format %{ "mov $dst, $src\t# ptr -> long" %}
8094
8095 ins_encode %{
8096 if ($dst$$reg != $src$$reg) {
8097 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8098 }
8099 %}
8100
8101 ins_pipe(ialu_reg);
8102 %}
8103
8104 // Convert oop into int for vectors alignment masking
8105 instruct convP2I(iRegINoSp dst, iRegP src) %{
8106 match(Set dst (ConvL2I (CastP2X src)));
8107
8108 ins_cost(INSN_COST);
8109 format %{ "movw $dst, $src\t# ptr -> int" %}
8110 ins_encode %{
8111 __ movw($dst$$Register, $src$$Register);
8112 %}
8113
8114 ins_pipe(ialu_reg);
8115 %}
8116
8117 // Convert compressed oop into int for vectors alignment masking
8118 // in case of 32bit oops (heap < 4Gb).
8119 instruct convN2I(iRegINoSp dst, iRegN src)
8120 %{
8121 predicate(CompressedOops::shift() == 0);
8122 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8123
8124 ins_cost(INSN_COST);
8125 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8126 ins_encode %{
8127 __ movw($dst$$Register, $src$$Register);
8128 %}
8129
8130 ins_pipe(ialu_reg);
8131 %}
8132
8133
8134 // Convert oop pointer into compressed form
8135 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8136 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8137 match(Set dst (EncodeP src));
8138 effect(KILL cr);
8139 ins_cost(INSN_COST * 3);
8140 format %{ "encode_heap_oop $dst, $src" %}
8141 ins_encode %{
8142 Register s = $src$$Register;
8143 Register d = $dst$$Register;
8144 __ encode_heap_oop(d, s);
8145 %}
8146 ins_pipe(ialu_reg);
8147 %}
8148
8149 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8150 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8151 match(Set dst (EncodeP src));
8152 ins_cost(INSN_COST * 3);
8153 format %{ "encode_heap_oop_not_null $dst, $src" %}
8154 ins_encode %{
8155 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8156 %}
8157 ins_pipe(ialu_reg);
8158 %}
8159
8160 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8161 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8162 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8163 match(Set dst (DecodeN src));
8164 ins_cost(INSN_COST * 3);
8165 format %{ "decode_heap_oop $dst, $src" %}
8166 ins_encode %{
8167 Register s = $src$$Register;
8168 Register d = $dst$$Register;
8169 __ decode_heap_oop(d, s);
8170 %}
8171 ins_pipe(ialu_reg);
8172 %}
8173
8174 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8175 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8176 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8177 match(Set dst (DecodeN src));
8178 ins_cost(INSN_COST * 3);
8179 format %{ "decode_heap_oop_not_null $dst, $src" %}
8180 ins_encode %{
8181 Register s = $src$$Register;
8182 Register d = $dst$$Register;
8183 __ decode_heap_oop_not_null(d, s);
8184 %}
8185 ins_pipe(ialu_reg);
8186 %}
8187
8188 // n.b. AArch64 implementations of encode_klass_not_null and
8189 // decode_klass_not_null do not modify the flags register so, unlike
8190 // Intel, we don't kill CR as a side effect here
8191
8192 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8193 match(Set dst (EncodePKlass src));
8194
8195 ins_cost(INSN_COST * 3);
8196 format %{ "encode_klass_not_null $dst,$src" %}
8197
8198 ins_encode %{
8199 Register src_reg = as_Register($src$$reg);
8200 Register dst_reg = as_Register($dst$$reg);
8201 __ encode_klass_not_null(dst_reg, src_reg);
8202 %}
8203
8204 ins_pipe(ialu_reg);
8205 %}
8206
8207 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8208 match(Set dst (DecodeNKlass src));
8209
8210 ins_cost(INSN_COST * 3);
8211 format %{ "decode_klass_not_null $dst,$src" %}
8212
8213 ins_encode %{
8214 Register src_reg = as_Register($src$$reg);
8215 Register dst_reg = as_Register($dst$$reg);
8216 if (dst_reg != src_reg) {
8217 __ decode_klass_not_null(dst_reg, src_reg);
8218 } else {
8219 __ decode_klass_not_null(dst_reg);
8220 }
8221 %}
8222
8223 ins_pipe(ialu_reg);
8224 %}
8225
8226 instruct checkCastPP(iRegPNoSp dst)
8227 %{
8228 match(Set dst (CheckCastPP dst));
8229
8230 size(0);
8231 format %{ "# checkcastPP of $dst" %}
8232 ins_encode(/* empty encoding */);
8233 ins_pipe(pipe_class_empty);
8234 %}
8235
8236 instruct castPP(iRegPNoSp dst)
8237 %{
8238 match(Set dst (CastPP dst));
8239
8240 size(0);
8241 format %{ "# castPP of $dst" %}
8242 ins_encode(/* empty encoding */);
8243 ins_pipe(pipe_class_empty);
8244 %}
8245
8246 instruct castII(iRegI dst)
8247 %{
8248 predicate(VerifyConstraintCasts == 0);
8249 match(Set dst (CastII dst));
8250
8251 size(0);
8252 format %{ "# castII of $dst" %}
8253 ins_encode(/* empty encoding */);
8254 ins_cost(0);
8255 ins_pipe(pipe_class_empty);
8256 %}
8257
8258 instruct castII_checked(iRegI dst, rFlagsReg cr)
8259 %{
8260 predicate(VerifyConstraintCasts > 0);
8261 match(Set dst (CastII dst));
8262 effect(KILL cr);
8263
8264 format %{ "# castII_checked of $dst" %}
8265 ins_encode %{
8266 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8267 %}
8268 ins_pipe(pipe_slow);
8269 %}
8270
8271 instruct castLL(iRegL dst)
8272 %{
8273 predicate(VerifyConstraintCasts == 0);
8274 match(Set dst (CastLL dst));
8275
8276 size(0);
8277 format %{ "# castLL of $dst" %}
8278 ins_encode(/* empty encoding */);
8279 ins_cost(0);
8280 ins_pipe(pipe_class_empty);
8281 %}
8282
8283 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8284 %{
8285 predicate(VerifyConstraintCasts > 0);
8286 match(Set dst (CastLL dst));
8287 effect(KILL cr);
8288
8289 format %{ "# castLL_checked of $dst" %}
8290 ins_encode %{
8291 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8292 %}
8293 ins_pipe(pipe_slow);
8294 %}
8295
8296 instruct castHH(vRegF dst)
8297 %{
8298 match(Set dst (CastHH dst));
8299 size(0);
8300 format %{ "# castHH of $dst" %}
8301 ins_encode(/* empty encoding */);
8302 ins_cost(0);
8303 ins_pipe(pipe_class_empty);
8304 %}
8305
8306 instruct castFF(vRegF dst)
8307 %{
8308 match(Set dst (CastFF dst));
8309
8310 size(0);
8311 format %{ "# castFF of $dst" %}
8312 ins_encode(/* empty encoding */);
8313 ins_cost(0);
8314 ins_pipe(pipe_class_empty);
8315 %}
8316
8317 instruct castDD(vRegD dst)
8318 %{
8319 match(Set dst (CastDD dst));
8320
8321 size(0);
8322 format %{ "# castDD of $dst" %}
8323 ins_encode(/* empty encoding */);
8324 ins_cost(0);
8325 ins_pipe(pipe_class_empty);
8326 %}
8327
8328 instruct castVV(vReg dst)
8329 %{
8330 match(Set dst (CastVV dst));
8331
8332 size(0);
8333 format %{ "# castVV of $dst" %}
8334 ins_encode(/* empty encoding */);
8335 ins_cost(0);
8336 ins_pipe(pipe_class_empty);
8337 %}
8338
8339 instruct castVVMask(pRegGov dst)
8340 %{
8341 match(Set dst (CastVV dst));
8342
8343 size(0);
8344 format %{ "# castVV of $dst" %}
8345 ins_encode(/* empty encoding */);
8346 ins_cost(0);
8347 ins_pipe(pipe_class_empty);
8348 %}
8349
8350 // Manifest a CmpU result in an integer register.
8351 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8352 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8353 %{
8354 match(Set dst (CmpU3 src1 src2));
8355 effect(KILL flags);
8356
8357 ins_cost(INSN_COST * 3);
8358 format %{
8359 "cmpw $src1, $src2\n\t"
8360 "csetw $dst, ne\n\t"
8361 "cnegw $dst, lo\t# CmpU3(reg)"
8362 %}
8363 ins_encode %{
8364 __ cmpw($src1$$Register, $src2$$Register);
8365 __ csetw($dst$$Register, Assembler::NE);
8366 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8367 %}
8368
8369 ins_pipe(pipe_class_default);
8370 %}
8371
8372 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8373 %{
8374 match(Set dst (CmpU3 src1 src2));
8375 effect(KILL flags);
8376
8377 ins_cost(INSN_COST * 3);
8378 format %{
8379 "subsw zr, $src1, $src2\n\t"
8380 "csetw $dst, ne\n\t"
8381 "cnegw $dst, lo\t# CmpU3(imm)"
8382 %}
8383 ins_encode %{
8384 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8385 __ csetw($dst$$Register, Assembler::NE);
8386 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8387 %}
8388
8389 ins_pipe(pipe_class_default);
8390 %}
8391
8392 // Manifest a CmpUL result in an integer register.
8393 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8394 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8395 %{
8396 match(Set dst (CmpUL3 src1 src2));
8397 effect(KILL flags);
8398
8399 ins_cost(INSN_COST * 3);
8400 format %{
8401 "cmp $src1, $src2\n\t"
8402 "csetw $dst, ne\n\t"
8403 "cnegw $dst, lo\t# CmpUL3(reg)"
8404 %}
8405 ins_encode %{
8406 __ cmp($src1$$Register, $src2$$Register);
8407 __ csetw($dst$$Register, Assembler::NE);
8408 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8409 %}
8410
8411 ins_pipe(pipe_class_default);
8412 %}
8413
8414 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8415 %{
8416 match(Set dst (CmpUL3 src1 src2));
8417 effect(KILL flags);
8418
8419 ins_cost(INSN_COST * 3);
8420 format %{
8421 "subs zr, $src1, $src2\n\t"
8422 "csetw $dst, ne\n\t"
8423 "cnegw $dst, lo\t# CmpUL3(imm)"
8424 %}
8425 ins_encode %{
8426 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8427 __ csetw($dst$$Register, Assembler::NE);
8428 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8429 %}
8430
8431 ins_pipe(pipe_class_default);
8432 %}
8433
8434 // Manifest a CmpL result in an integer register.
8435 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8436 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8437 %{
8438 match(Set dst (CmpL3 src1 src2));
8439 effect(KILL flags);
8440
8441 ins_cost(INSN_COST * 3);
8442 format %{
8443 "cmp $src1, $src2\n\t"
8444 "csetw $dst, ne\n\t"
8445 "cnegw $dst, lt\t# CmpL3(reg)"
8446 %}
8447 ins_encode %{
8448 __ cmp($src1$$Register, $src2$$Register);
8449 __ csetw($dst$$Register, Assembler::NE);
8450 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8451 %}
8452
8453 ins_pipe(pipe_class_default);
8454 %}
8455
8456 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8457 %{
8458 match(Set dst (CmpL3 src1 src2));
8459 effect(KILL flags);
8460
8461 ins_cost(INSN_COST * 3);
8462 format %{
8463 "subs zr, $src1, $src2\n\t"
8464 "csetw $dst, ne\n\t"
8465 "cnegw $dst, lt\t# CmpL3(imm)"
8466 %}
8467 ins_encode %{
8468 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8469 __ csetw($dst$$Register, Assembler::NE);
8470 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8471 %}
8472
8473 ins_pipe(pipe_class_default);
8474 %}
8475
8476 // ============================================================================
8477 // Conditional Move Instructions
8478
8479 // n.b. we have identical rules for both a signed compare op (cmpOp)
8480 // and an unsigned compare op (cmpOpU). it would be nice if we could
8481 // define an op class which merged both inputs and use it to type the
8482 // argument to a single rule. unfortunatelyt his fails because the
8483 // opclass does not live up to the COND_INTER interface of its
8484 // component operands. When the generic code tries to negate the
8485 // operand it ends up running the generci Machoper::negate method
8486 // which throws a ShouldNotHappen. So, we have to provide two flavours
8487 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8488
8489 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8490 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8491
8492 ins_cost(INSN_COST * 2);
8493 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8494
8495 ins_encode %{
8496 __ cselw(as_Register($dst$$reg),
8497 as_Register($src2$$reg),
8498 as_Register($src1$$reg),
8499 (Assembler::Condition)$cmp$$cmpcode);
8500 %}
8501
8502 ins_pipe(icond_reg_reg);
8503 %}
8504
8505 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8506 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8507
8508 ins_cost(INSN_COST * 2);
8509 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8510
8511 ins_encode %{
8512 __ cselw(as_Register($dst$$reg),
8513 as_Register($src2$$reg),
8514 as_Register($src1$$reg),
8515 (Assembler::Condition)$cmp$$cmpcode);
8516 %}
8517
8518 ins_pipe(icond_reg_reg);
8519 %}
8520
8521 // special cases where one arg is zero
8522
8523 // n.b. this is selected in preference to the rule above because it
8524 // avoids loading constant 0 into a source register
8525
8526 // TODO
8527 // we ought only to be able to cull one of these variants as the ideal
8528 // transforms ought always to order the zero consistently (to left/right?)
8529
8530 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8531 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8532
8533 ins_cost(INSN_COST * 2);
8534 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8535
8536 ins_encode %{
8537 __ cselw(as_Register($dst$$reg),
8538 as_Register($src$$reg),
8539 zr,
8540 (Assembler::Condition)$cmp$$cmpcode);
8541 %}
8542
8543 ins_pipe(icond_reg);
8544 %}
8545
8546 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8547 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8548
8549 ins_cost(INSN_COST * 2);
8550 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8551
8552 ins_encode %{
8553 __ cselw(as_Register($dst$$reg),
8554 as_Register($src$$reg),
8555 zr,
8556 (Assembler::Condition)$cmp$$cmpcode);
8557 %}
8558
8559 ins_pipe(icond_reg);
8560 %}
8561
8562 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8563 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8564
8565 ins_cost(INSN_COST * 2);
8566 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8567
8568 ins_encode %{
8569 __ cselw(as_Register($dst$$reg),
8570 zr,
8571 as_Register($src$$reg),
8572 (Assembler::Condition)$cmp$$cmpcode);
8573 %}
8574
8575 ins_pipe(icond_reg);
8576 %}
8577
8578 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8579 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8580
8581 ins_cost(INSN_COST * 2);
8582 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8583
8584 ins_encode %{
8585 __ cselw(as_Register($dst$$reg),
8586 zr,
8587 as_Register($src$$reg),
8588 (Assembler::Condition)$cmp$$cmpcode);
8589 %}
8590
8591 ins_pipe(icond_reg);
8592 %}
8593
8594 // special case for creating a boolean 0 or 1
8595
8596 // n.b. this is selected in preference to the rule above because it
8597 // avoids loading constants 0 and 1 into a source register
8598
8599 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8600 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8601
8602 ins_cost(INSN_COST * 2);
8603 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8604
8605 ins_encode %{
8606 // equivalently
8607 // cset(as_Register($dst$$reg),
8608 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8609 __ csincw(as_Register($dst$$reg),
8610 zr,
8611 zr,
8612 (Assembler::Condition)$cmp$$cmpcode);
8613 %}
8614
8615 ins_pipe(icond_none);
8616 %}
8617
8618 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8619 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8620
8621 ins_cost(INSN_COST * 2);
8622 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8623
8624 ins_encode %{
8625 // equivalently
8626 // cset(as_Register($dst$$reg),
8627 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8628 __ csincw(as_Register($dst$$reg),
8629 zr,
8630 zr,
8631 (Assembler::Condition)$cmp$$cmpcode);
8632 %}
8633
8634 ins_pipe(icond_none);
8635 %}
8636
8637 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8638 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8639
8640 ins_cost(INSN_COST * 2);
8641 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8642
8643 ins_encode %{
8644 __ csel(as_Register($dst$$reg),
8645 as_Register($src2$$reg),
8646 as_Register($src1$$reg),
8647 (Assembler::Condition)$cmp$$cmpcode);
8648 %}
8649
8650 ins_pipe(icond_reg_reg);
8651 %}
8652
8653 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8654 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8655
8656 ins_cost(INSN_COST * 2);
8657 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8658
8659 ins_encode %{
8660 __ csel(as_Register($dst$$reg),
8661 as_Register($src2$$reg),
8662 as_Register($src1$$reg),
8663 (Assembler::Condition)$cmp$$cmpcode);
8664 %}
8665
8666 ins_pipe(icond_reg_reg);
8667 %}
8668
8669 // special cases where one arg is zero
8670
8671 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8672 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8673
8674 ins_cost(INSN_COST * 2);
8675 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8676
8677 ins_encode %{
8678 __ csel(as_Register($dst$$reg),
8679 zr,
8680 as_Register($src$$reg),
8681 (Assembler::Condition)$cmp$$cmpcode);
8682 %}
8683
8684 ins_pipe(icond_reg);
8685 %}
8686
8687 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8688 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8689
8690 ins_cost(INSN_COST * 2);
8691 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8692
8693 ins_encode %{
8694 __ csel(as_Register($dst$$reg),
8695 zr,
8696 as_Register($src$$reg),
8697 (Assembler::Condition)$cmp$$cmpcode);
8698 %}
8699
8700 ins_pipe(icond_reg);
8701 %}
8702
8703 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8704 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8705
8706 ins_cost(INSN_COST * 2);
8707 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8708
8709 ins_encode %{
8710 __ csel(as_Register($dst$$reg),
8711 as_Register($src$$reg),
8712 zr,
8713 (Assembler::Condition)$cmp$$cmpcode);
8714 %}
8715
8716 ins_pipe(icond_reg);
8717 %}
8718
8719 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8720 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8721
8722 ins_cost(INSN_COST * 2);
8723 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8724
8725 ins_encode %{
8726 __ csel(as_Register($dst$$reg),
8727 as_Register($src$$reg),
8728 zr,
8729 (Assembler::Condition)$cmp$$cmpcode);
8730 %}
8731
8732 ins_pipe(icond_reg);
8733 %}
8734
8735 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8736 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8737
8738 ins_cost(INSN_COST * 2);
8739 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8740
8741 ins_encode %{
8742 __ csel(as_Register($dst$$reg),
8743 as_Register($src2$$reg),
8744 as_Register($src1$$reg),
8745 (Assembler::Condition)$cmp$$cmpcode);
8746 %}
8747
8748 ins_pipe(icond_reg_reg);
8749 %}
8750
8751 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8752 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8753
8754 ins_cost(INSN_COST * 2);
8755 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8756
8757 ins_encode %{
8758 __ csel(as_Register($dst$$reg),
8759 as_Register($src2$$reg),
8760 as_Register($src1$$reg),
8761 (Assembler::Condition)$cmp$$cmpcode);
8762 %}
8763
8764 ins_pipe(icond_reg_reg);
8765 %}
8766
8767 // special cases where one arg is zero
8768
8769 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8770 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8771
8772 ins_cost(INSN_COST * 2);
8773 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8774
8775 ins_encode %{
8776 __ csel(as_Register($dst$$reg),
8777 zr,
8778 as_Register($src$$reg),
8779 (Assembler::Condition)$cmp$$cmpcode);
8780 %}
8781
8782 ins_pipe(icond_reg);
8783 %}
8784
8785 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8786 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8787
8788 ins_cost(INSN_COST * 2);
8789 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8790
8791 ins_encode %{
8792 __ csel(as_Register($dst$$reg),
8793 zr,
8794 as_Register($src$$reg),
8795 (Assembler::Condition)$cmp$$cmpcode);
8796 %}
8797
8798 ins_pipe(icond_reg);
8799 %}
8800
8801 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8802 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8803
8804 ins_cost(INSN_COST * 2);
8805 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8806
8807 ins_encode %{
8808 __ csel(as_Register($dst$$reg),
8809 as_Register($src$$reg),
8810 zr,
8811 (Assembler::Condition)$cmp$$cmpcode);
8812 %}
8813
8814 ins_pipe(icond_reg);
8815 %}
8816
8817 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8818 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8819
8820 ins_cost(INSN_COST * 2);
8821 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8822
8823 ins_encode %{
8824 __ csel(as_Register($dst$$reg),
8825 as_Register($src$$reg),
8826 zr,
8827 (Assembler::Condition)$cmp$$cmpcode);
8828 %}
8829
8830 ins_pipe(icond_reg);
8831 %}
8832
8833 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8834 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8835
8836 ins_cost(INSN_COST * 2);
8837 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8838
8839 ins_encode %{
8840 __ cselw(as_Register($dst$$reg),
8841 as_Register($src2$$reg),
8842 as_Register($src1$$reg),
8843 (Assembler::Condition)$cmp$$cmpcode);
8844 %}
8845
8846 ins_pipe(icond_reg_reg);
8847 %}
8848
8849 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8850 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8851
8852 ins_cost(INSN_COST * 2);
8853 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8854
8855 ins_encode %{
8856 __ cselw(as_Register($dst$$reg),
8857 as_Register($src2$$reg),
8858 as_Register($src1$$reg),
8859 (Assembler::Condition)$cmp$$cmpcode);
8860 %}
8861
8862 ins_pipe(icond_reg_reg);
8863 %}
8864
8865 // special cases where one arg is zero
8866
8867 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8868 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8869
8870 ins_cost(INSN_COST * 2);
8871 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8872
8873 ins_encode %{
8874 __ cselw(as_Register($dst$$reg),
8875 zr,
8876 as_Register($src$$reg),
8877 (Assembler::Condition)$cmp$$cmpcode);
8878 %}
8879
8880 ins_pipe(icond_reg);
8881 %}
8882
8883 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8884 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8885
8886 ins_cost(INSN_COST * 2);
8887 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8888
8889 ins_encode %{
8890 __ cselw(as_Register($dst$$reg),
8891 zr,
8892 as_Register($src$$reg),
8893 (Assembler::Condition)$cmp$$cmpcode);
8894 %}
8895
8896 ins_pipe(icond_reg);
8897 %}
8898
8899 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8900 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8901
8902 ins_cost(INSN_COST * 2);
8903 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8904
8905 ins_encode %{
8906 __ cselw(as_Register($dst$$reg),
8907 as_Register($src$$reg),
8908 zr,
8909 (Assembler::Condition)$cmp$$cmpcode);
8910 %}
8911
8912 ins_pipe(icond_reg);
8913 %}
8914
8915 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8916 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8917
8918 ins_cost(INSN_COST * 2);
8919 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8920
8921 ins_encode %{
8922 __ cselw(as_Register($dst$$reg),
8923 as_Register($src$$reg),
8924 zr,
8925 (Assembler::Condition)$cmp$$cmpcode);
8926 %}
8927
8928 ins_pipe(icond_reg);
8929 %}
8930
8931 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8932 %{
8933 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8934
8935 ins_cost(INSN_COST * 3);
8936
8937 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8938 ins_encode %{
8939 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8940 __ fcsels(as_FloatRegister($dst$$reg),
8941 as_FloatRegister($src2$$reg),
8942 as_FloatRegister($src1$$reg),
8943 cond);
8944 %}
8945
8946 ins_pipe(fp_cond_reg_reg_s);
8947 %}
8948
8949 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8950 %{
8951 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8952
8953 ins_cost(INSN_COST * 3);
8954
8955 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8956 ins_encode %{
8957 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8958 __ fcsels(as_FloatRegister($dst$$reg),
8959 as_FloatRegister($src2$$reg),
8960 as_FloatRegister($src1$$reg),
8961 cond);
8962 %}
8963
8964 ins_pipe(fp_cond_reg_reg_s);
8965 %}
8966
8967 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8968 %{
8969 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8970
8971 ins_cost(INSN_COST * 3);
8972
8973 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8974 ins_encode %{
8975 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8976 __ fcseld(as_FloatRegister($dst$$reg),
8977 as_FloatRegister($src2$$reg),
8978 as_FloatRegister($src1$$reg),
8979 cond);
8980 %}
8981
8982 ins_pipe(fp_cond_reg_reg_d);
8983 %}
8984
8985 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8986 %{
8987 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8988
8989 ins_cost(INSN_COST * 3);
8990
8991 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8992 ins_encode %{
8993 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8994 __ fcseld(as_FloatRegister($dst$$reg),
8995 as_FloatRegister($src2$$reg),
8996 as_FloatRegister($src1$$reg),
8997 cond);
8998 %}
8999
9000 ins_pipe(fp_cond_reg_reg_d);
9001 %}
9002
9003 // ============================================================================
9004 // Arithmetic Instructions
9005 //
9006
9007 // Integer Addition
9008
9009 // TODO
9010 // these currently employ operations which do not set CR and hence are
9011 // not flagged as killing CR but we would like to isolate the cases
9012 // where we want to set flags from those where we don't. need to work
9013 // out how to do that.
9014
9015 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9016 match(Set dst (AddI src1 src2));
9017
9018 ins_cost(INSN_COST);
9019 format %{ "addw $dst, $src1, $src2" %}
9020
9021 ins_encode %{
9022 __ addw(as_Register($dst$$reg),
9023 as_Register($src1$$reg),
9024 as_Register($src2$$reg));
9025 %}
9026
9027 ins_pipe(ialu_reg_reg);
9028 %}
9029
9030 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9031 match(Set dst (AddI src1 src2));
9032
9033 ins_cost(INSN_COST);
9034 format %{ "addw $dst, $src1, $src2" %}
9035
9036 // use opcode to indicate that this is an add not a sub
9037 opcode(0x0);
9038
9039 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9040
9041 ins_pipe(ialu_reg_imm);
9042 %}
9043
9044 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9045 match(Set dst (AddI (ConvL2I src1) src2));
9046
9047 ins_cost(INSN_COST);
9048 format %{ "addw $dst, $src1, $src2" %}
9049
9050 // use opcode to indicate that this is an add not a sub
9051 opcode(0x0);
9052
9053 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9054
9055 ins_pipe(ialu_reg_imm);
9056 %}
9057
9058 // Pointer Addition
9059 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9060 match(Set dst (AddP src1 src2));
9061
9062 ins_cost(INSN_COST);
9063 format %{ "add $dst, $src1, $src2\t# ptr" %}
9064
9065 ins_encode %{
9066 __ add(as_Register($dst$$reg),
9067 as_Register($src1$$reg),
9068 as_Register($src2$$reg));
9069 %}
9070
9071 ins_pipe(ialu_reg_reg);
9072 %}
9073
9074 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9075 match(Set dst (AddP src1 (ConvI2L src2)));
9076
9077 ins_cost(1.9 * INSN_COST);
9078 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9079
9080 ins_encode %{
9081 __ add(as_Register($dst$$reg),
9082 as_Register($src1$$reg),
9083 as_Register($src2$$reg), ext::sxtw);
9084 %}
9085
9086 ins_pipe(ialu_reg_reg);
9087 %}
9088
9089 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9090 match(Set dst (AddP src1 (LShiftL src2 scale)));
9091
9092 ins_cost(1.9 * INSN_COST);
9093 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9094
9095 ins_encode %{
9096 __ lea(as_Register($dst$$reg),
9097 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9098 Address::lsl($scale$$constant)));
9099 %}
9100
9101 ins_pipe(ialu_reg_reg_shift);
9102 %}
9103
9104 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9105 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9106
9107 ins_cost(1.9 * INSN_COST);
9108 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9109
9110 ins_encode %{
9111 __ lea(as_Register($dst$$reg),
9112 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9113 Address::sxtw($scale$$constant)));
9114 %}
9115
9116 ins_pipe(ialu_reg_reg_shift);
9117 %}
9118
9119 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9120 match(Set dst (LShiftL (ConvI2L src) scale));
9121
9122 ins_cost(INSN_COST);
9123 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9124
9125 ins_encode %{
9126 __ sbfiz(as_Register($dst$$reg),
9127 as_Register($src$$reg),
9128 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9129 %}
9130
9131 ins_pipe(ialu_reg_shift);
9132 %}
9133
9134 // Pointer Immediate Addition
9135 // n.b. this needs to be more expensive than using an indirect memory
9136 // operand
9137 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9138 match(Set dst (AddP src1 src2));
9139
9140 ins_cost(INSN_COST);
9141 format %{ "add $dst, $src1, $src2\t# ptr" %}
9142
9143 // use opcode to indicate that this is an add not a sub
9144 opcode(0x0);
9145
9146 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9147
9148 ins_pipe(ialu_reg_imm);
9149 %}
9150
9151 // Long Addition
9152 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9153
9154 match(Set dst (AddL src1 src2));
9155
9156 ins_cost(INSN_COST);
9157 format %{ "add $dst, $src1, $src2" %}
9158
9159 ins_encode %{
9160 __ add(as_Register($dst$$reg),
9161 as_Register($src1$$reg),
9162 as_Register($src2$$reg));
9163 %}
9164
9165 ins_pipe(ialu_reg_reg);
9166 %}
9167
9168 // No constant pool entries requiredLong Immediate Addition.
9169 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9170 match(Set dst (AddL src1 src2));
9171
9172 ins_cost(INSN_COST);
9173 format %{ "add $dst, $src1, $src2" %}
9174
9175 // use opcode to indicate that this is an add not a sub
9176 opcode(0x0);
9177
9178 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9179
9180 ins_pipe(ialu_reg_imm);
9181 %}
9182
9183 // Integer Subtraction
9184 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9185 match(Set dst (SubI src1 src2));
9186
9187 ins_cost(INSN_COST);
9188 format %{ "subw $dst, $src1, $src2" %}
9189
9190 ins_encode %{
9191 __ subw(as_Register($dst$$reg),
9192 as_Register($src1$$reg),
9193 as_Register($src2$$reg));
9194 %}
9195
9196 ins_pipe(ialu_reg_reg);
9197 %}
9198
9199 // Immediate Subtraction
9200 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9201 match(Set dst (SubI src1 src2));
9202
9203 ins_cost(INSN_COST);
9204 format %{ "subw $dst, $src1, $src2" %}
9205
9206 // use opcode to indicate that this is a sub not an add
9207 opcode(0x1);
9208
9209 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9210
9211 ins_pipe(ialu_reg_imm);
9212 %}
9213
9214 // Long Subtraction
9215 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9216
9217 match(Set dst (SubL src1 src2));
9218
9219 ins_cost(INSN_COST);
9220 format %{ "sub $dst, $src1, $src2" %}
9221
9222 ins_encode %{
9223 __ sub(as_Register($dst$$reg),
9224 as_Register($src1$$reg),
9225 as_Register($src2$$reg));
9226 %}
9227
9228 ins_pipe(ialu_reg_reg);
9229 %}
9230
9231 // No constant pool entries requiredLong Immediate Subtraction.
9232 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9233 match(Set dst (SubL src1 src2));
9234
9235 ins_cost(INSN_COST);
9236 format %{ "sub$dst, $src1, $src2" %}
9237
9238 // use opcode to indicate that this is a sub not an add
9239 opcode(0x1);
9240
9241 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9242
9243 ins_pipe(ialu_reg_imm);
9244 %}
9245
9246 // Integer Negation (special case for sub)
9247
9248 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9249 match(Set dst (SubI zero src));
9250
9251 ins_cost(INSN_COST);
9252 format %{ "negw $dst, $src\t# int" %}
9253
9254 ins_encode %{
9255 __ negw(as_Register($dst$$reg),
9256 as_Register($src$$reg));
9257 %}
9258
9259 ins_pipe(ialu_reg);
9260 %}
9261
9262 // Long Negation
9263
9264 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9265 match(Set dst (SubL zero src));
9266
9267 ins_cost(INSN_COST);
9268 format %{ "neg $dst, $src\t# long" %}
9269
9270 ins_encode %{
9271 __ neg(as_Register($dst$$reg),
9272 as_Register($src$$reg));
9273 %}
9274
9275 ins_pipe(ialu_reg);
9276 %}
9277
9278 // Integer Multiply
9279
9280 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9281 match(Set dst (MulI src1 src2));
9282
9283 ins_cost(INSN_COST * 3);
9284 format %{ "mulw $dst, $src1, $src2" %}
9285
9286 ins_encode %{
9287 __ mulw(as_Register($dst$$reg),
9288 as_Register($src1$$reg),
9289 as_Register($src2$$reg));
9290 %}
9291
9292 ins_pipe(imul_reg_reg);
9293 %}
9294
9295 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9296 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9297
9298 ins_cost(INSN_COST * 3);
9299 format %{ "smull $dst, $src1, $src2" %}
9300
9301 ins_encode %{
9302 __ smull(as_Register($dst$$reg),
9303 as_Register($src1$$reg),
9304 as_Register($src2$$reg));
9305 %}
9306
9307 ins_pipe(imul_reg_reg);
9308 %}
9309
9310 // Long Multiply
9311
9312 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9313 match(Set dst (MulL src1 src2));
9314
9315 ins_cost(INSN_COST * 5);
9316 format %{ "mul $dst, $src1, $src2" %}
9317
9318 ins_encode %{
9319 __ mul(as_Register($dst$$reg),
9320 as_Register($src1$$reg),
9321 as_Register($src2$$reg));
9322 %}
9323
9324 ins_pipe(lmul_reg_reg);
9325 %}
9326
9327 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9328 %{
9329 match(Set dst (MulHiL src1 src2));
9330
9331 ins_cost(INSN_COST * 7);
9332 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9333
9334 ins_encode %{
9335 __ smulh(as_Register($dst$$reg),
9336 as_Register($src1$$reg),
9337 as_Register($src2$$reg));
9338 %}
9339
9340 ins_pipe(lmul_reg_reg);
9341 %}
9342
9343 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9344 %{
9345 match(Set dst (UMulHiL src1 src2));
9346
9347 ins_cost(INSN_COST * 7);
9348 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9349
9350 ins_encode %{
9351 __ umulh(as_Register($dst$$reg),
9352 as_Register($src1$$reg),
9353 as_Register($src2$$reg));
9354 %}
9355
9356 ins_pipe(lmul_reg_reg);
9357 %}
9358
9359 // Combined Integer Multiply & Add/Sub
9360
9361 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9362 match(Set dst (AddI src3 (MulI src1 src2)));
9363
9364 ins_cost(INSN_COST * 3);
9365 format %{ "madd $dst, $src1, $src2, $src3" %}
9366
9367 ins_encode %{
9368 __ maddw(as_Register($dst$$reg),
9369 as_Register($src1$$reg),
9370 as_Register($src2$$reg),
9371 as_Register($src3$$reg));
9372 %}
9373
9374 ins_pipe(imac_reg_reg);
9375 %}
9376
9377 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9378 match(Set dst (SubI src3 (MulI src1 src2)));
9379
9380 ins_cost(INSN_COST * 3);
9381 format %{ "msub $dst, $src1, $src2, $src3" %}
9382
9383 ins_encode %{
9384 __ msubw(as_Register($dst$$reg),
9385 as_Register($src1$$reg),
9386 as_Register($src2$$reg),
9387 as_Register($src3$$reg));
9388 %}
9389
9390 ins_pipe(imac_reg_reg);
9391 %}
9392
9393 // Combined Integer Multiply & Neg
9394
9395 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9396 match(Set dst (MulI (SubI zero src1) src2));
9397
9398 ins_cost(INSN_COST * 3);
9399 format %{ "mneg $dst, $src1, $src2" %}
9400
9401 ins_encode %{
9402 __ mnegw(as_Register($dst$$reg),
9403 as_Register($src1$$reg),
9404 as_Register($src2$$reg));
9405 %}
9406
9407 ins_pipe(imac_reg_reg);
9408 %}
9409
9410 // Combined Long Multiply & Add/Sub
9411
9412 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9413 match(Set dst (AddL src3 (MulL src1 src2)));
9414
9415 ins_cost(INSN_COST * 5);
9416 format %{ "madd $dst, $src1, $src2, $src3" %}
9417
9418 ins_encode %{
9419 __ madd(as_Register($dst$$reg),
9420 as_Register($src1$$reg),
9421 as_Register($src2$$reg),
9422 as_Register($src3$$reg));
9423 %}
9424
9425 ins_pipe(lmac_reg_reg);
9426 %}
9427
9428 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9429 match(Set dst (SubL src3 (MulL src1 src2)));
9430
9431 ins_cost(INSN_COST * 5);
9432 format %{ "msub $dst, $src1, $src2, $src3" %}
9433
9434 ins_encode %{
9435 __ msub(as_Register($dst$$reg),
9436 as_Register($src1$$reg),
9437 as_Register($src2$$reg),
9438 as_Register($src3$$reg));
9439 %}
9440
9441 ins_pipe(lmac_reg_reg);
9442 %}
9443
9444 // Combined Long Multiply & Neg
9445
9446 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9447 match(Set dst (MulL (SubL zero src1) src2));
9448
9449 ins_cost(INSN_COST * 5);
9450 format %{ "mneg $dst, $src1, $src2" %}
9451
9452 ins_encode %{
9453 __ mneg(as_Register($dst$$reg),
9454 as_Register($src1$$reg),
9455 as_Register($src2$$reg));
9456 %}
9457
9458 ins_pipe(lmac_reg_reg);
9459 %}
9460
9461 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9462
9463 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9464 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9465
9466 ins_cost(INSN_COST * 3);
9467 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9468
9469 ins_encode %{
9470 __ smaddl(as_Register($dst$$reg),
9471 as_Register($src1$$reg),
9472 as_Register($src2$$reg),
9473 as_Register($src3$$reg));
9474 %}
9475
9476 ins_pipe(imac_reg_reg);
9477 %}
9478
9479 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9480 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9481
9482 ins_cost(INSN_COST * 3);
9483 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9484
9485 ins_encode %{
9486 __ smsubl(as_Register($dst$$reg),
9487 as_Register($src1$$reg),
9488 as_Register($src2$$reg),
9489 as_Register($src3$$reg));
9490 %}
9491
9492 ins_pipe(imac_reg_reg);
9493 %}
9494
9495 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9496 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9497
9498 ins_cost(INSN_COST * 3);
9499 format %{ "smnegl $dst, $src1, $src2" %}
9500
9501 ins_encode %{
9502 __ smnegl(as_Register($dst$$reg),
9503 as_Register($src1$$reg),
9504 as_Register($src2$$reg));
9505 %}
9506
9507 ins_pipe(imac_reg_reg);
9508 %}
9509
9510 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9511
9512 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9513 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9514
9515 ins_cost(INSN_COST * 5);
9516 format %{ "mulw rscratch1, $src1, $src2\n\t"
9517 "maddw $dst, $src3, $src4, rscratch1" %}
9518
9519 ins_encode %{
9520 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9521 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9522
9523 ins_pipe(imac_reg_reg);
9524 %}
9525
9526 // Integer Divide
9527
9528 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9529 match(Set dst (DivI src1 src2));
9530
9531 ins_cost(INSN_COST * 19);
9532 format %{ "sdivw $dst, $src1, $src2" %}
9533
9534 ins_encode(aarch64_enc_divw(dst, src1, src2));
9535 ins_pipe(idiv_reg_reg);
9536 %}
9537
9538 // Long Divide
9539
9540 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9541 match(Set dst (DivL src1 src2));
9542
9543 ins_cost(INSN_COST * 35);
9544 format %{ "sdiv $dst, $src1, $src2" %}
9545
9546 ins_encode(aarch64_enc_div(dst, src1, src2));
9547 ins_pipe(ldiv_reg_reg);
9548 %}
9549
9550 // Integer Remainder
9551
9552 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9553 match(Set dst (ModI src1 src2));
9554
9555 ins_cost(INSN_COST * 22);
9556 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9557 "msubw $dst, rscratch1, $src2, $src1" %}
9558
9559 ins_encode(aarch64_enc_modw(dst, src1, src2));
9560 ins_pipe(idiv_reg_reg);
9561 %}
9562
9563 // Long Remainder
9564
9565 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9566 match(Set dst (ModL src1 src2));
9567
9568 ins_cost(INSN_COST * 38);
9569 format %{ "sdiv rscratch1, $src1, $src2\n"
9570 "msub $dst, rscratch1, $src2, $src1" %}
9571
9572 ins_encode(aarch64_enc_mod(dst, src1, src2));
9573 ins_pipe(ldiv_reg_reg);
9574 %}
9575
9576 // Unsigned Integer Divide
9577
9578 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9579 match(Set dst (UDivI src1 src2));
9580
9581 ins_cost(INSN_COST * 19);
9582 format %{ "udivw $dst, $src1, $src2" %}
9583
9584 ins_encode %{
9585 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9586 %}
9587
9588 ins_pipe(idiv_reg_reg);
9589 %}
9590
9591 // Unsigned Long Divide
9592
9593 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9594 match(Set dst (UDivL src1 src2));
9595
9596 ins_cost(INSN_COST * 35);
9597 format %{ "udiv $dst, $src1, $src2" %}
9598
9599 ins_encode %{
9600 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9601 %}
9602
9603 ins_pipe(ldiv_reg_reg);
9604 %}
9605
9606 // Unsigned Integer Remainder
9607
9608 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9609 match(Set dst (UModI src1 src2));
9610
9611 ins_cost(INSN_COST * 22);
9612 format %{ "udivw rscratch1, $src1, $src2\n\t"
9613 "msubw $dst, rscratch1, $src2, $src1" %}
9614
9615 ins_encode %{
9616 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9617 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9618 %}
9619
9620 ins_pipe(idiv_reg_reg);
9621 %}
9622
9623 // Unsigned Long Remainder
9624
9625 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9626 match(Set dst (UModL src1 src2));
9627
9628 ins_cost(INSN_COST * 38);
9629 format %{ "udiv rscratch1, $src1, $src2\n"
9630 "msub $dst, rscratch1, $src2, $src1" %}
9631
9632 ins_encode %{
9633 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9634 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9635 %}
9636
9637 ins_pipe(ldiv_reg_reg);
9638 %}
9639
9640 // Integer Shifts
9641
9642 // Shift Left Register
9643 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9644 match(Set dst (LShiftI src1 src2));
9645
9646 ins_cost(INSN_COST * 2);
9647 format %{ "lslvw $dst, $src1, $src2" %}
9648
9649 ins_encode %{
9650 __ lslvw(as_Register($dst$$reg),
9651 as_Register($src1$$reg),
9652 as_Register($src2$$reg));
9653 %}
9654
9655 ins_pipe(ialu_reg_reg_vshift);
9656 %}
9657
9658 // Shift Left Immediate
9659 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9660 match(Set dst (LShiftI src1 src2));
9661
9662 ins_cost(INSN_COST);
9663 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9664
9665 ins_encode %{
9666 __ lslw(as_Register($dst$$reg),
9667 as_Register($src1$$reg),
9668 $src2$$constant & 0x1f);
9669 %}
9670
9671 ins_pipe(ialu_reg_shift);
9672 %}
9673
9674 // Shift Right Logical Register
9675 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9676 match(Set dst (URShiftI src1 src2));
9677
9678 ins_cost(INSN_COST * 2);
9679 format %{ "lsrvw $dst, $src1, $src2" %}
9680
9681 ins_encode %{
9682 __ lsrvw(as_Register($dst$$reg),
9683 as_Register($src1$$reg),
9684 as_Register($src2$$reg));
9685 %}
9686
9687 ins_pipe(ialu_reg_reg_vshift);
9688 %}
9689
9690 // Shift Right Logical Immediate
9691 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9692 match(Set dst (URShiftI src1 src2));
9693
9694 ins_cost(INSN_COST);
9695 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9696
9697 ins_encode %{
9698 __ lsrw(as_Register($dst$$reg),
9699 as_Register($src1$$reg),
9700 $src2$$constant & 0x1f);
9701 %}
9702
9703 ins_pipe(ialu_reg_shift);
9704 %}
9705
9706 // Shift Right Arithmetic Register
9707 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9708 match(Set dst (RShiftI src1 src2));
9709
9710 ins_cost(INSN_COST * 2);
9711 format %{ "asrvw $dst, $src1, $src2" %}
9712
9713 ins_encode %{
9714 __ asrvw(as_Register($dst$$reg),
9715 as_Register($src1$$reg),
9716 as_Register($src2$$reg));
9717 %}
9718
9719 ins_pipe(ialu_reg_reg_vshift);
9720 %}
9721
9722 // Shift Right Arithmetic Immediate
9723 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9724 match(Set dst (RShiftI src1 src2));
9725
9726 ins_cost(INSN_COST);
9727 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9728
9729 ins_encode %{
9730 __ asrw(as_Register($dst$$reg),
9731 as_Register($src1$$reg),
9732 $src2$$constant & 0x1f);
9733 %}
9734
9735 ins_pipe(ialu_reg_shift);
9736 %}
9737
9738 // Combined Int Mask and Right Shift (using UBFM)
9739 // TODO
9740
9741 // Long Shifts
9742
9743 // Shift Left Register
9744 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9745 match(Set dst (LShiftL src1 src2));
9746
9747 ins_cost(INSN_COST * 2);
9748 format %{ "lslv $dst, $src1, $src2" %}
9749
9750 ins_encode %{
9751 __ lslv(as_Register($dst$$reg),
9752 as_Register($src1$$reg),
9753 as_Register($src2$$reg));
9754 %}
9755
9756 ins_pipe(ialu_reg_reg_vshift);
9757 %}
9758
9759 // Shift Left Immediate
9760 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9761 match(Set dst (LShiftL src1 src2));
9762
9763 ins_cost(INSN_COST);
9764 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9765
9766 ins_encode %{
9767 __ lsl(as_Register($dst$$reg),
9768 as_Register($src1$$reg),
9769 $src2$$constant & 0x3f);
9770 %}
9771
9772 ins_pipe(ialu_reg_shift);
9773 %}
9774
9775 // Shift Right Logical Register
9776 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9777 match(Set dst (URShiftL src1 src2));
9778
9779 ins_cost(INSN_COST * 2);
9780 format %{ "lsrv $dst, $src1, $src2" %}
9781
9782 ins_encode %{
9783 __ lsrv(as_Register($dst$$reg),
9784 as_Register($src1$$reg),
9785 as_Register($src2$$reg));
9786 %}
9787
9788 ins_pipe(ialu_reg_reg_vshift);
9789 %}
9790
9791 // Shift Right Logical Immediate
9792 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9793 match(Set dst (URShiftL src1 src2));
9794
9795 ins_cost(INSN_COST);
9796 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9797
9798 ins_encode %{
9799 __ lsr(as_Register($dst$$reg),
9800 as_Register($src1$$reg),
9801 $src2$$constant & 0x3f);
9802 %}
9803
9804 ins_pipe(ialu_reg_shift);
9805 %}
9806
9807 // A special-case pattern for card table stores.
9808 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9809 match(Set dst (URShiftL (CastP2X src1) src2));
9810
9811 ins_cost(INSN_COST);
9812 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9813
9814 ins_encode %{
9815 __ lsr(as_Register($dst$$reg),
9816 as_Register($src1$$reg),
9817 $src2$$constant & 0x3f);
9818 %}
9819
9820 ins_pipe(ialu_reg_shift);
9821 %}
9822
9823 // Shift Right Arithmetic Register
9824 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9825 match(Set dst (RShiftL src1 src2));
9826
9827 ins_cost(INSN_COST * 2);
9828 format %{ "asrv $dst, $src1, $src2" %}
9829
9830 ins_encode %{
9831 __ asrv(as_Register($dst$$reg),
9832 as_Register($src1$$reg),
9833 as_Register($src2$$reg));
9834 %}
9835
9836 ins_pipe(ialu_reg_reg_vshift);
9837 %}
9838
9839 // Shift Right Arithmetic Immediate
9840 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9841 match(Set dst (RShiftL src1 src2));
9842
9843 ins_cost(INSN_COST);
9844 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9845
9846 ins_encode %{
9847 __ asr(as_Register($dst$$reg),
9848 as_Register($src1$$reg),
9849 $src2$$constant & 0x3f);
9850 %}
9851
9852 ins_pipe(ialu_reg_shift);
9853 %}
9854
9855 // BEGIN This section of the file is automatically generated. Do not edit --------------
9856 // This section is generated from aarch64_ad.m4
9857
9858 // This pattern is automatically generated from aarch64_ad.m4
9859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9860 instruct regL_not_reg(iRegLNoSp dst,
9861 iRegL src1, immL_M1 m1,
9862 rFlagsReg cr) %{
9863 match(Set dst (XorL src1 m1));
9864 ins_cost(INSN_COST);
9865 format %{ "eon $dst, $src1, zr" %}
9866
9867 ins_encode %{
9868 __ eon(as_Register($dst$$reg),
9869 as_Register($src1$$reg),
9870 zr,
9871 Assembler::LSL, 0);
9872 %}
9873
9874 ins_pipe(ialu_reg);
9875 %}
9876
9877 // This pattern is automatically generated from aarch64_ad.m4
9878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9879 instruct regI_not_reg(iRegINoSp dst,
9880 iRegIorL2I src1, immI_M1 m1,
9881 rFlagsReg cr) %{
9882 match(Set dst (XorI src1 m1));
9883 ins_cost(INSN_COST);
9884 format %{ "eonw $dst, $src1, zr" %}
9885
9886 ins_encode %{
9887 __ eonw(as_Register($dst$$reg),
9888 as_Register($src1$$reg),
9889 zr,
9890 Assembler::LSL, 0);
9891 %}
9892
9893 ins_pipe(ialu_reg);
9894 %}
9895
9896 // This pattern is automatically generated from aarch64_ad.m4
9897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9898 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9899 immI0 zero, iRegIorL2I src1, immI src2) %{
9900 match(Set dst (SubI zero (URShiftI src1 src2)));
9901
9902 ins_cost(1.9 * INSN_COST);
9903 format %{ "negw $dst, $src1, LSR $src2" %}
9904
9905 ins_encode %{
9906 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9907 Assembler::LSR, $src2$$constant & 0x1f);
9908 %}
9909
9910 ins_pipe(ialu_reg_shift);
9911 %}
9912
9913 // This pattern is automatically generated from aarch64_ad.m4
9914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9915 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9916 immI0 zero, iRegIorL2I src1, immI src2) %{
9917 match(Set dst (SubI zero (RShiftI src1 src2)));
9918
9919 ins_cost(1.9 * INSN_COST);
9920 format %{ "negw $dst, $src1, ASR $src2" %}
9921
9922 ins_encode %{
9923 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9924 Assembler::ASR, $src2$$constant & 0x1f);
9925 %}
9926
9927 ins_pipe(ialu_reg_shift);
9928 %}
9929
9930 // This pattern is automatically generated from aarch64_ad.m4
9931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9932 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9933 immI0 zero, iRegIorL2I src1, immI src2) %{
9934 match(Set dst (SubI zero (LShiftI src1 src2)));
9935
9936 ins_cost(1.9 * INSN_COST);
9937 format %{ "negw $dst, $src1, LSL $src2" %}
9938
9939 ins_encode %{
9940 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9941 Assembler::LSL, $src2$$constant & 0x1f);
9942 %}
9943
9944 ins_pipe(ialu_reg_shift);
9945 %}
9946
9947 // This pattern is automatically generated from aarch64_ad.m4
9948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9949 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9950 immL0 zero, iRegL src1, immI src2) %{
9951 match(Set dst (SubL zero (URShiftL src1 src2)));
9952
9953 ins_cost(1.9 * INSN_COST);
9954 format %{ "neg $dst, $src1, LSR $src2" %}
9955
9956 ins_encode %{
9957 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9958 Assembler::LSR, $src2$$constant & 0x3f);
9959 %}
9960
9961 ins_pipe(ialu_reg_shift);
9962 %}
9963
9964 // This pattern is automatically generated from aarch64_ad.m4
9965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9966 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9967 immL0 zero, iRegL src1, immI src2) %{
9968 match(Set dst (SubL zero (RShiftL src1 src2)));
9969
9970 ins_cost(1.9 * INSN_COST);
9971 format %{ "neg $dst, $src1, ASR $src2" %}
9972
9973 ins_encode %{
9974 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9975 Assembler::ASR, $src2$$constant & 0x3f);
9976 %}
9977
9978 ins_pipe(ialu_reg_shift);
9979 %}
9980
9981 // This pattern is automatically generated from aarch64_ad.m4
9982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9983 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9984 immL0 zero, iRegL src1, immI src2) %{
9985 match(Set dst (SubL zero (LShiftL src1 src2)));
9986
9987 ins_cost(1.9 * INSN_COST);
9988 format %{ "neg $dst, $src1, LSL $src2" %}
9989
9990 ins_encode %{
9991 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9992 Assembler::LSL, $src2$$constant & 0x3f);
9993 %}
9994
9995 ins_pipe(ialu_reg_shift);
9996 %}
9997
9998 // This pattern is automatically generated from aarch64_ad.m4
9999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10000 instruct AndI_reg_not_reg(iRegINoSp dst,
10001 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10002 match(Set dst (AndI src1 (XorI src2 m1)));
10003 ins_cost(INSN_COST);
10004 format %{ "bicw $dst, $src1, $src2" %}
10005
10006 ins_encode %{
10007 __ bicw(as_Register($dst$$reg),
10008 as_Register($src1$$reg),
10009 as_Register($src2$$reg),
10010 Assembler::LSL, 0);
10011 %}
10012
10013 ins_pipe(ialu_reg_reg);
10014 %}
10015
10016 // This pattern is automatically generated from aarch64_ad.m4
10017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10018 instruct AndL_reg_not_reg(iRegLNoSp dst,
10019 iRegL src1, iRegL src2, immL_M1 m1) %{
10020 match(Set dst (AndL src1 (XorL src2 m1)));
10021 ins_cost(INSN_COST);
10022 format %{ "bic $dst, $src1, $src2" %}
10023
10024 ins_encode %{
10025 __ bic(as_Register($dst$$reg),
10026 as_Register($src1$$reg),
10027 as_Register($src2$$reg),
10028 Assembler::LSL, 0);
10029 %}
10030
10031 ins_pipe(ialu_reg_reg);
10032 %}
10033
10034 // This pattern is automatically generated from aarch64_ad.m4
10035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10036 instruct OrI_reg_not_reg(iRegINoSp dst,
10037 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10038 match(Set dst (OrI src1 (XorI src2 m1)));
10039 ins_cost(INSN_COST);
10040 format %{ "ornw $dst, $src1, $src2" %}
10041
10042 ins_encode %{
10043 __ ornw(as_Register($dst$$reg),
10044 as_Register($src1$$reg),
10045 as_Register($src2$$reg),
10046 Assembler::LSL, 0);
10047 %}
10048
10049 ins_pipe(ialu_reg_reg);
10050 %}
10051
10052 // This pattern is automatically generated from aarch64_ad.m4
10053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10054 instruct OrL_reg_not_reg(iRegLNoSp dst,
10055 iRegL src1, iRegL src2, immL_M1 m1) %{
10056 match(Set dst (OrL src1 (XorL src2 m1)));
10057 ins_cost(INSN_COST);
10058 format %{ "orn $dst, $src1, $src2" %}
10059
10060 ins_encode %{
10061 __ orn(as_Register($dst$$reg),
10062 as_Register($src1$$reg),
10063 as_Register($src2$$reg),
10064 Assembler::LSL, 0);
10065 %}
10066
10067 ins_pipe(ialu_reg_reg);
10068 %}
10069
10070 // This pattern is automatically generated from aarch64_ad.m4
10071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10072 instruct XorI_reg_not_reg(iRegINoSp dst,
10073 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10074 match(Set dst (XorI m1 (XorI src2 src1)));
10075 ins_cost(INSN_COST);
10076 format %{ "eonw $dst, $src1, $src2" %}
10077
10078 ins_encode %{
10079 __ eonw(as_Register($dst$$reg),
10080 as_Register($src1$$reg),
10081 as_Register($src2$$reg),
10082 Assembler::LSL, 0);
10083 %}
10084
10085 ins_pipe(ialu_reg_reg);
10086 %}
10087
10088 // This pattern is automatically generated from aarch64_ad.m4
10089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10090 instruct XorL_reg_not_reg(iRegLNoSp dst,
10091 iRegL src1, iRegL src2, immL_M1 m1) %{
10092 match(Set dst (XorL m1 (XorL src2 src1)));
10093 ins_cost(INSN_COST);
10094 format %{ "eon $dst, $src1, $src2" %}
10095
10096 ins_encode %{
10097 __ eon(as_Register($dst$$reg),
10098 as_Register($src1$$reg),
10099 as_Register($src2$$reg),
10100 Assembler::LSL, 0);
10101 %}
10102
10103 ins_pipe(ialu_reg_reg);
10104 %}
10105
10106 // This pattern is automatically generated from aarch64_ad.m4
10107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10108 // val & (-1 ^ (val >>> shift)) ==> bicw
10109 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10110 iRegIorL2I src1, iRegIorL2I src2,
10111 immI src3, immI_M1 src4) %{
10112 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10113 ins_cost(1.9 * INSN_COST);
10114 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10115
10116 ins_encode %{
10117 __ bicw(as_Register($dst$$reg),
10118 as_Register($src1$$reg),
10119 as_Register($src2$$reg),
10120 Assembler::LSR,
10121 $src3$$constant & 0x1f);
10122 %}
10123
10124 ins_pipe(ialu_reg_reg_shift);
10125 %}
10126
10127 // This pattern is automatically generated from aarch64_ad.m4
10128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10129 // val & (-1 ^ (val >>> shift)) ==> bic
10130 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10131 iRegL src1, iRegL src2,
10132 immI src3, immL_M1 src4) %{
10133 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10134 ins_cost(1.9 * INSN_COST);
10135 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10136
10137 ins_encode %{
10138 __ bic(as_Register($dst$$reg),
10139 as_Register($src1$$reg),
10140 as_Register($src2$$reg),
10141 Assembler::LSR,
10142 $src3$$constant & 0x3f);
10143 %}
10144
10145 ins_pipe(ialu_reg_reg_shift);
10146 %}
10147
10148 // This pattern is automatically generated from aarch64_ad.m4
10149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10150 // val & (-1 ^ (val >> shift)) ==> bicw
10151 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10152 iRegIorL2I src1, iRegIorL2I src2,
10153 immI src3, immI_M1 src4) %{
10154 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10155 ins_cost(1.9 * INSN_COST);
10156 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10157
10158 ins_encode %{
10159 __ bicw(as_Register($dst$$reg),
10160 as_Register($src1$$reg),
10161 as_Register($src2$$reg),
10162 Assembler::ASR,
10163 $src3$$constant & 0x1f);
10164 %}
10165
10166 ins_pipe(ialu_reg_reg_shift);
10167 %}
10168
10169 // This pattern is automatically generated from aarch64_ad.m4
10170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10171 // val & (-1 ^ (val >> shift)) ==> bic
10172 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10173 iRegL src1, iRegL src2,
10174 immI src3, immL_M1 src4) %{
10175 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10176 ins_cost(1.9 * INSN_COST);
10177 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10178
10179 ins_encode %{
10180 __ bic(as_Register($dst$$reg),
10181 as_Register($src1$$reg),
10182 as_Register($src2$$reg),
10183 Assembler::ASR,
10184 $src3$$constant & 0x3f);
10185 %}
10186
10187 ins_pipe(ialu_reg_reg_shift);
10188 %}
10189
10190 // This pattern is automatically generated from aarch64_ad.m4
10191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10192 // val & (-1 ^ (val ror shift)) ==> bicw
10193 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10194 iRegIorL2I src1, iRegIorL2I src2,
10195 immI src3, immI_M1 src4) %{
10196 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10197 ins_cost(1.9 * INSN_COST);
10198 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10199
10200 ins_encode %{
10201 __ bicw(as_Register($dst$$reg),
10202 as_Register($src1$$reg),
10203 as_Register($src2$$reg),
10204 Assembler::ROR,
10205 $src3$$constant & 0x1f);
10206 %}
10207
10208 ins_pipe(ialu_reg_reg_shift);
10209 %}
10210
10211 // This pattern is automatically generated from aarch64_ad.m4
10212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10213 // val & (-1 ^ (val ror shift)) ==> bic
10214 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10215 iRegL src1, iRegL src2,
10216 immI src3, immL_M1 src4) %{
10217 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10218 ins_cost(1.9 * INSN_COST);
10219 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10220
10221 ins_encode %{
10222 __ bic(as_Register($dst$$reg),
10223 as_Register($src1$$reg),
10224 as_Register($src2$$reg),
10225 Assembler::ROR,
10226 $src3$$constant & 0x3f);
10227 %}
10228
10229 ins_pipe(ialu_reg_reg_shift);
10230 %}
10231
10232 // This pattern is automatically generated from aarch64_ad.m4
10233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10234 // val & (-1 ^ (val << shift)) ==> bicw
10235 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10236 iRegIorL2I src1, iRegIorL2I src2,
10237 immI src3, immI_M1 src4) %{
10238 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10239 ins_cost(1.9 * INSN_COST);
10240 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10241
10242 ins_encode %{
10243 __ bicw(as_Register($dst$$reg),
10244 as_Register($src1$$reg),
10245 as_Register($src2$$reg),
10246 Assembler::LSL,
10247 $src3$$constant & 0x1f);
10248 %}
10249
10250 ins_pipe(ialu_reg_reg_shift);
10251 %}
10252
10253 // This pattern is automatically generated from aarch64_ad.m4
10254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10255 // val & (-1 ^ (val << shift)) ==> bic
10256 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10257 iRegL src1, iRegL src2,
10258 immI src3, immL_M1 src4) %{
10259 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10260 ins_cost(1.9 * INSN_COST);
10261 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10262
10263 ins_encode %{
10264 __ bic(as_Register($dst$$reg),
10265 as_Register($src1$$reg),
10266 as_Register($src2$$reg),
10267 Assembler::LSL,
10268 $src3$$constant & 0x3f);
10269 %}
10270
10271 ins_pipe(ialu_reg_reg_shift);
10272 %}
10273
10274 // This pattern is automatically generated from aarch64_ad.m4
10275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10276 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10277 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10278 iRegIorL2I src1, iRegIorL2I src2,
10279 immI src3, immI_M1 src4) %{
10280 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10281 ins_cost(1.9 * INSN_COST);
10282 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10283
10284 ins_encode %{
10285 __ eonw(as_Register($dst$$reg),
10286 as_Register($src1$$reg),
10287 as_Register($src2$$reg),
10288 Assembler::LSR,
10289 $src3$$constant & 0x1f);
10290 %}
10291
10292 ins_pipe(ialu_reg_reg_shift);
10293 %}
10294
10295 // This pattern is automatically generated from aarch64_ad.m4
10296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10297 // val ^ (-1 ^ (val >>> shift)) ==> eon
10298 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10299 iRegL src1, iRegL src2,
10300 immI src3, immL_M1 src4) %{
10301 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10302 ins_cost(1.9 * INSN_COST);
10303 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10304
10305 ins_encode %{
10306 __ eon(as_Register($dst$$reg),
10307 as_Register($src1$$reg),
10308 as_Register($src2$$reg),
10309 Assembler::LSR,
10310 $src3$$constant & 0x3f);
10311 %}
10312
10313 ins_pipe(ialu_reg_reg_shift);
10314 %}
10315
10316 // This pattern is automatically generated from aarch64_ad.m4
10317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10318 // val ^ (-1 ^ (val >> shift)) ==> eonw
10319 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10320 iRegIorL2I src1, iRegIorL2I src2,
10321 immI src3, immI_M1 src4) %{
10322 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10323 ins_cost(1.9 * INSN_COST);
10324 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10325
10326 ins_encode %{
10327 __ eonw(as_Register($dst$$reg),
10328 as_Register($src1$$reg),
10329 as_Register($src2$$reg),
10330 Assembler::ASR,
10331 $src3$$constant & 0x1f);
10332 %}
10333
10334 ins_pipe(ialu_reg_reg_shift);
10335 %}
10336
10337 // This pattern is automatically generated from aarch64_ad.m4
10338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10339 // val ^ (-1 ^ (val >> shift)) ==> eon
10340 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10341 iRegL src1, iRegL src2,
10342 immI src3, immL_M1 src4) %{
10343 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10344 ins_cost(1.9 * INSN_COST);
10345 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10346
10347 ins_encode %{
10348 __ eon(as_Register($dst$$reg),
10349 as_Register($src1$$reg),
10350 as_Register($src2$$reg),
10351 Assembler::ASR,
10352 $src3$$constant & 0x3f);
10353 %}
10354
10355 ins_pipe(ialu_reg_reg_shift);
10356 %}
10357
10358 // This pattern is automatically generated from aarch64_ad.m4
10359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10360 // val ^ (-1 ^ (val ror shift)) ==> eonw
10361 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10362 iRegIorL2I src1, iRegIorL2I src2,
10363 immI src3, immI_M1 src4) %{
10364 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10365 ins_cost(1.9 * INSN_COST);
10366 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10367
10368 ins_encode %{
10369 __ eonw(as_Register($dst$$reg),
10370 as_Register($src1$$reg),
10371 as_Register($src2$$reg),
10372 Assembler::ROR,
10373 $src3$$constant & 0x1f);
10374 %}
10375
10376 ins_pipe(ialu_reg_reg_shift);
10377 %}
10378
10379 // This pattern is automatically generated from aarch64_ad.m4
10380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10381 // val ^ (-1 ^ (val ror shift)) ==> eon
10382 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10383 iRegL src1, iRegL src2,
10384 immI src3, immL_M1 src4) %{
10385 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10386 ins_cost(1.9 * INSN_COST);
10387 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10388
10389 ins_encode %{
10390 __ eon(as_Register($dst$$reg),
10391 as_Register($src1$$reg),
10392 as_Register($src2$$reg),
10393 Assembler::ROR,
10394 $src3$$constant & 0x3f);
10395 %}
10396
10397 ins_pipe(ialu_reg_reg_shift);
10398 %}
10399
10400 // This pattern is automatically generated from aarch64_ad.m4
10401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10402 // val ^ (-1 ^ (val << shift)) ==> eonw
10403 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10404 iRegIorL2I src1, iRegIorL2I src2,
10405 immI src3, immI_M1 src4) %{
10406 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10407 ins_cost(1.9 * INSN_COST);
10408 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10409
10410 ins_encode %{
10411 __ eonw(as_Register($dst$$reg),
10412 as_Register($src1$$reg),
10413 as_Register($src2$$reg),
10414 Assembler::LSL,
10415 $src3$$constant & 0x1f);
10416 %}
10417
10418 ins_pipe(ialu_reg_reg_shift);
10419 %}
10420
10421 // This pattern is automatically generated from aarch64_ad.m4
10422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10423 // val ^ (-1 ^ (val << shift)) ==> eon
10424 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10425 iRegL src1, iRegL src2,
10426 immI src3, immL_M1 src4) %{
10427 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10428 ins_cost(1.9 * INSN_COST);
10429 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10430
10431 ins_encode %{
10432 __ eon(as_Register($dst$$reg),
10433 as_Register($src1$$reg),
10434 as_Register($src2$$reg),
10435 Assembler::LSL,
10436 $src3$$constant & 0x3f);
10437 %}
10438
10439 ins_pipe(ialu_reg_reg_shift);
10440 %}
10441
10442 // This pattern is automatically generated from aarch64_ad.m4
10443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10444 // val | (-1 ^ (val >>> shift)) ==> ornw
10445 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10446 iRegIorL2I src1, iRegIorL2I src2,
10447 immI src3, immI_M1 src4) %{
10448 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10449 ins_cost(1.9 * INSN_COST);
10450 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10451
10452 ins_encode %{
10453 __ ornw(as_Register($dst$$reg),
10454 as_Register($src1$$reg),
10455 as_Register($src2$$reg),
10456 Assembler::LSR,
10457 $src3$$constant & 0x1f);
10458 %}
10459
10460 ins_pipe(ialu_reg_reg_shift);
10461 %}
10462
10463 // This pattern is automatically generated from aarch64_ad.m4
10464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10465 // val | (-1 ^ (val >>> shift)) ==> orn
10466 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10467 iRegL src1, iRegL src2,
10468 immI src3, immL_M1 src4) %{
10469 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10470 ins_cost(1.9 * INSN_COST);
10471 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10472
10473 ins_encode %{
10474 __ orn(as_Register($dst$$reg),
10475 as_Register($src1$$reg),
10476 as_Register($src2$$reg),
10477 Assembler::LSR,
10478 $src3$$constant & 0x3f);
10479 %}
10480
10481 ins_pipe(ialu_reg_reg_shift);
10482 %}
10483
10484 // This pattern is automatically generated from aarch64_ad.m4
10485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10486 // val | (-1 ^ (val >> shift)) ==> ornw
10487 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10488 iRegIorL2I src1, iRegIorL2I src2,
10489 immI src3, immI_M1 src4) %{
10490 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10491 ins_cost(1.9 * INSN_COST);
10492 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10493
10494 ins_encode %{
10495 __ ornw(as_Register($dst$$reg),
10496 as_Register($src1$$reg),
10497 as_Register($src2$$reg),
10498 Assembler::ASR,
10499 $src3$$constant & 0x1f);
10500 %}
10501
10502 ins_pipe(ialu_reg_reg_shift);
10503 %}
10504
10505 // This pattern is automatically generated from aarch64_ad.m4
10506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10507 // val | (-1 ^ (val >> shift)) ==> orn
10508 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10509 iRegL src1, iRegL src2,
10510 immI src3, immL_M1 src4) %{
10511 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10512 ins_cost(1.9 * INSN_COST);
10513 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10514
10515 ins_encode %{
10516 __ orn(as_Register($dst$$reg),
10517 as_Register($src1$$reg),
10518 as_Register($src2$$reg),
10519 Assembler::ASR,
10520 $src3$$constant & 0x3f);
10521 %}
10522
10523 ins_pipe(ialu_reg_reg_shift);
10524 %}
10525
10526 // This pattern is automatically generated from aarch64_ad.m4
10527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10528 // val | (-1 ^ (val ror shift)) ==> ornw
10529 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10530 iRegIorL2I src1, iRegIorL2I src2,
10531 immI src3, immI_M1 src4) %{
10532 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10533 ins_cost(1.9 * INSN_COST);
10534 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10535
10536 ins_encode %{
10537 __ ornw(as_Register($dst$$reg),
10538 as_Register($src1$$reg),
10539 as_Register($src2$$reg),
10540 Assembler::ROR,
10541 $src3$$constant & 0x1f);
10542 %}
10543
10544 ins_pipe(ialu_reg_reg_shift);
10545 %}
10546
10547 // This pattern is automatically generated from aarch64_ad.m4
10548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10549 // val | (-1 ^ (val ror shift)) ==> orn
10550 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10551 iRegL src1, iRegL src2,
10552 immI src3, immL_M1 src4) %{
10553 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10554 ins_cost(1.9 * INSN_COST);
10555 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10556
10557 ins_encode %{
10558 __ orn(as_Register($dst$$reg),
10559 as_Register($src1$$reg),
10560 as_Register($src2$$reg),
10561 Assembler::ROR,
10562 $src3$$constant & 0x3f);
10563 %}
10564
10565 ins_pipe(ialu_reg_reg_shift);
10566 %}
10567
10568 // This pattern is automatically generated from aarch64_ad.m4
10569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10570 // val | (-1 ^ (val << shift)) ==> ornw
10571 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10572 iRegIorL2I src1, iRegIorL2I src2,
10573 immI src3, immI_M1 src4) %{
10574 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10575 ins_cost(1.9 * INSN_COST);
10576 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10577
10578 ins_encode %{
10579 __ ornw(as_Register($dst$$reg),
10580 as_Register($src1$$reg),
10581 as_Register($src2$$reg),
10582 Assembler::LSL,
10583 $src3$$constant & 0x1f);
10584 %}
10585
10586 ins_pipe(ialu_reg_reg_shift);
10587 %}
10588
10589 // This pattern is automatically generated from aarch64_ad.m4
10590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10591 // val | (-1 ^ (val << shift)) ==> orn
10592 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10593 iRegL src1, iRegL src2,
10594 immI src3, immL_M1 src4) %{
10595 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10596 ins_cost(1.9 * INSN_COST);
10597 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10598
10599 ins_encode %{
10600 __ orn(as_Register($dst$$reg),
10601 as_Register($src1$$reg),
10602 as_Register($src2$$reg),
10603 Assembler::LSL,
10604 $src3$$constant & 0x3f);
10605 %}
10606
10607 ins_pipe(ialu_reg_reg_shift);
10608 %}
10609
10610 // This pattern is automatically generated from aarch64_ad.m4
10611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10612 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10613 iRegIorL2I src1, iRegIorL2I src2,
10614 immI src3) %{
10615 match(Set dst (AndI src1 (URShiftI src2 src3)));
10616
10617 ins_cost(1.9 * INSN_COST);
10618 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10619
10620 ins_encode %{
10621 __ andw(as_Register($dst$$reg),
10622 as_Register($src1$$reg),
10623 as_Register($src2$$reg),
10624 Assembler::LSR,
10625 $src3$$constant & 0x1f);
10626 %}
10627
10628 ins_pipe(ialu_reg_reg_shift);
10629 %}
10630
10631 // This pattern is automatically generated from aarch64_ad.m4
10632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10633 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10634 iRegL src1, iRegL src2,
10635 immI src3) %{
10636 match(Set dst (AndL src1 (URShiftL src2 src3)));
10637
10638 ins_cost(1.9 * INSN_COST);
10639 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10640
10641 ins_encode %{
10642 __ andr(as_Register($dst$$reg),
10643 as_Register($src1$$reg),
10644 as_Register($src2$$reg),
10645 Assembler::LSR,
10646 $src3$$constant & 0x3f);
10647 %}
10648
10649 ins_pipe(ialu_reg_reg_shift);
10650 %}
10651
10652 // This pattern is automatically generated from aarch64_ad.m4
10653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10654 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10655 iRegIorL2I src1, iRegIorL2I src2,
10656 immI src3) %{
10657 match(Set dst (AndI src1 (RShiftI src2 src3)));
10658
10659 ins_cost(1.9 * INSN_COST);
10660 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10661
10662 ins_encode %{
10663 __ andw(as_Register($dst$$reg),
10664 as_Register($src1$$reg),
10665 as_Register($src2$$reg),
10666 Assembler::ASR,
10667 $src3$$constant & 0x1f);
10668 %}
10669
10670 ins_pipe(ialu_reg_reg_shift);
10671 %}
10672
10673 // This pattern is automatically generated from aarch64_ad.m4
10674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10675 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10676 iRegL src1, iRegL src2,
10677 immI src3) %{
10678 match(Set dst (AndL src1 (RShiftL src2 src3)));
10679
10680 ins_cost(1.9 * INSN_COST);
10681 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10682
10683 ins_encode %{
10684 __ andr(as_Register($dst$$reg),
10685 as_Register($src1$$reg),
10686 as_Register($src2$$reg),
10687 Assembler::ASR,
10688 $src3$$constant & 0x3f);
10689 %}
10690
10691 ins_pipe(ialu_reg_reg_shift);
10692 %}
10693
10694 // This pattern is automatically generated from aarch64_ad.m4
10695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10696 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10697 iRegIorL2I src1, iRegIorL2I src2,
10698 immI src3) %{
10699 match(Set dst (AndI src1 (LShiftI src2 src3)));
10700
10701 ins_cost(1.9 * INSN_COST);
10702 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10703
10704 ins_encode %{
10705 __ andw(as_Register($dst$$reg),
10706 as_Register($src1$$reg),
10707 as_Register($src2$$reg),
10708 Assembler::LSL,
10709 $src3$$constant & 0x1f);
10710 %}
10711
10712 ins_pipe(ialu_reg_reg_shift);
10713 %}
10714
10715 // This pattern is automatically generated from aarch64_ad.m4
10716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10717 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10718 iRegL src1, iRegL src2,
10719 immI src3) %{
10720 match(Set dst (AndL src1 (LShiftL src2 src3)));
10721
10722 ins_cost(1.9 * INSN_COST);
10723 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10724
10725 ins_encode %{
10726 __ andr(as_Register($dst$$reg),
10727 as_Register($src1$$reg),
10728 as_Register($src2$$reg),
10729 Assembler::LSL,
10730 $src3$$constant & 0x3f);
10731 %}
10732
10733 ins_pipe(ialu_reg_reg_shift);
10734 %}
10735
10736 // This pattern is automatically generated from aarch64_ad.m4
10737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10738 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10739 iRegIorL2I src1, iRegIorL2I src2,
10740 immI src3) %{
10741 match(Set dst (AndI src1 (RotateRight src2 src3)));
10742
10743 ins_cost(1.9 * INSN_COST);
10744 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10745
10746 ins_encode %{
10747 __ andw(as_Register($dst$$reg),
10748 as_Register($src1$$reg),
10749 as_Register($src2$$reg),
10750 Assembler::ROR,
10751 $src3$$constant & 0x1f);
10752 %}
10753
10754 ins_pipe(ialu_reg_reg_shift);
10755 %}
10756
10757 // This pattern is automatically generated from aarch64_ad.m4
10758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10759 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10760 iRegL src1, iRegL src2,
10761 immI src3) %{
10762 match(Set dst (AndL src1 (RotateRight src2 src3)));
10763
10764 ins_cost(1.9 * INSN_COST);
10765 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10766
10767 ins_encode %{
10768 __ andr(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 as_Register($src2$$reg),
10771 Assembler::ROR,
10772 $src3$$constant & 0x3f);
10773 %}
10774
10775 ins_pipe(ialu_reg_reg_shift);
10776 %}
10777
10778 // This pattern is automatically generated from aarch64_ad.m4
10779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10780 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10781 iRegIorL2I src1, iRegIorL2I src2,
10782 immI src3) %{
10783 match(Set dst (XorI src1 (URShiftI src2 src3)));
10784
10785 ins_cost(1.9 * INSN_COST);
10786 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10787
10788 ins_encode %{
10789 __ eorw(as_Register($dst$$reg),
10790 as_Register($src1$$reg),
10791 as_Register($src2$$reg),
10792 Assembler::LSR,
10793 $src3$$constant & 0x1f);
10794 %}
10795
10796 ins_pipe(ialu_reg_reg_shift);
10797 %}
10798
10799 // This pattern is automatically generated from aarch64_ad.m4
10800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10801 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10802 iRegL src1, iRegL src2,
10803 immI src3) %{
10804 match(Set dst (XorL src1 (URShiftL src2 src3)));
10805
10806 ins_cost(1.9 * INSN_COST);
10807 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10808
10809 ins_encode %{
10810 __ eor(as_Register($dst$$reg),
10811 as_Register($src1$$reg),
10812 as_Register($src2$$reg),
10813 Assembler::LSR,
10814 $src3$$constant & 0x3f);
10815 %}
10816
10817 ins_pipe(ialu_reg_reg_shift);
10818 %}
10819
10820 // This pattern is automatically generated from aarch64_ad.m4
10821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10822 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10823 iRegIorL2I src1, iRegIorL2I src2,
10824 immI src3) %{
10825 match(Set dst (XorI src1 (RShiftI src2 src3)));
10826
10827 ins_cost(1.9 * INSN_COST);
10828 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10829
10830 ins_encode %{
10831 __ eorw(as_Register($dst$$reg),
10832 as_Register($src1$$reg),
10833 as_Register($src2$$reg),
10834 Assembler::ASR,
10835 $src3$$constant & 0x1f);
10836 %}
10837
10838 ins_pipe(ialu_reg_reg_shift);
10839 %}
10840
10841 // This pattern is automatically generated from aarch64_ad.m4
10842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10843 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10844 iRegL src1, iRegL src2,
10845 immI src3) %{
10846 match(Set dst (XorL src1 (RShiftL src2 src3)));
10847
10848 ins_cost(1.9 * INSN_COST);
10849 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10850
10851 ins_encode %{
10852 __ eor(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 as_Register($src2$$reg),
10855 Assembler::ASR,
10856 $src3$$constant & 0x3f);
10857 %}
10858
10859 ins_pipe(ialu_reg_reg_shift);
10860 %}
10861
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10865 iRegIorL2I src1, iRegIorL2I src2,
10866 immI src3) %{
10867 match(Set dst (XorI src1 (LShiftI src2 src3)));
10868
10869 ins_cost(1.9 * INSN_COST);
10870 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10871
10872 ins_encode %{
10873 __ eorw(as_Register($dst$$reg),
10874 as_Register($src1$$reg),
10875 as_Register($src2$$reg),
10876 Assembler::LSL,
10877 $src3$$constant & 0x1f);
10878 %}
10879
10880 ins_pipe(ialu_reg_reg_shift);
10881 %}
10882
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10886 iRegL src1, iRegL src2,
10887 immI src3) %{
10888 match(Set dst (XorL src1 (LShiftL src2 src3)));
10889
10890 ins_cost(1.9 * INSN_COST);
10891 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10892
10893 ins_encode %{
10894 __ eor(as_Register($dst$$reg),
10895 as_Register($src1$$reg),
10896 as_Register($src2$$reg),
10897 Assembler::LSL,
10898 $src3$$constant & 0x3f);
10899 %}
10900
10901 ins_pipe(ialu_reg_reg_shift);
10902 %}
10903
10904 // This pattern is automatically generated from aarch64_ad.m4
10905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10906 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10907 iRegIorL2I src1, iRegIorL2I src2,
10908 immI src3) %{
10909 match(Set dst (XorI src1 (RotateRight src2 src3)));
10910
10911 ins_cost(1.9 * INSN_COST);
10912 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10913
10914 ins_encode %{
10915 __ eorw(as_Register($dst$$reg),
10916 as_Register($src1$$reg),
10917 as_Register($src2$$reg),
10918 Assembler::ROR,
10919 $src3$$constant & 0x1f);
10920 %}
10921
10922 ins_pipe(ialu_reg_reg_shift);
10923 %}
10924
10925 // This pattern is automatically generated from aarch64_ad.m4
10926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10927 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10928 iRegL src1, iRegL src2,
10929 immI src3) %{
10930 match(Set dst (XorL src1 (RotateRight src2 src3)));
10931
10932 ins_cost(1.9 * INSN_COST);
10933 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10934
10935 ins_encode %{
10936 __ eor(as_Register($dst$$reg),
10937 as_Register($src1$$reg),
10938 as_Register($src2$$reg),
10939 Assembler::ROR,
10940 $src3$$constant & 0x3f);
10941 %}
10942
10943 ins_pipe(ialu_reg_reg_shift);
10944 %}
10945
10946 // This pattern is automatically generated from aarch64_ad.m4
10947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10948 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10949 iRegIorL2I src1, iRegIorL2I src2,
10950 immI src3) %{
10951 match(Set dst (OrI src1 (URShiftI src2 src3)));
10952
10953 ins_cost(1.9 * INSN_COST);
10954 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10955
10956 ins_encode %{
10957 __ orrw(as_Register($dst$$reg),
10958 as_Register($src1$$reg),
10959 as_Register($src2$$reg),
10960 Assembler::LSR,
10961 $src3$$constant & 0x1f);
10962 %}
10963
10964 ins_pipe(ialu_reg_reg_shift);
10965 %}
10966
10967 // This pattern is automatically generated from aarch64_ad.m4
10968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10969 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10970 iRegL src1, iRegL src2,
10971 immI src3) %{
10972 match(Set dst (OrL src1 (URShiftL src2 src3)));
10973
10974 ins_cost(1.9 * INSN_COST);
10975 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10976
10977 ins_encode %{
10978 __ orr(as_Register($dst$$reg),
10979 as_Register($src1$$reg),
10980 as_Register($src2$$reg),
10981 Assembler::LSR,
10982 $src3$$constant & 0x3f);
10983 %}
10984
10985 ins_pipe(ialu_reg_reg_shift);
10986 %}
10987
10988 // This pattern is automatically generated from aarch64_ad.m4
10989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10990 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10991 iRegIorL2I src1, iRegIorL2I src2,
10992 immI src3) %{
10993 match(Set dst (OrI src1 (RShiftI src2 src3)));
10994
10995 ins_cost(1.9 * INSN_COST);
10996 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10997
10998 ins_encode %{
10999 __ orrw(as_Register($dst$$reg),
11000 as_Register($src1$$reg),
11001 as_Register($src2$$reg),
11002 Assembler::ASR,
11003 $src3$$constant & 0x1f);
11004 %}
11005
11006 ins_pipe(ialu_reg_reg_shift);
11007 %}
11008
11009 // This pattern is automatically generated from aarch64_ad.m4
11010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11011 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11012 iRegL src1, iRegL src2,
11013 immI src3) %{
11014 match(Set dst (OrL src1 (RShiftL src2 src3)));
11015
11016 ins_cost(1.9 * INSN_COST);
11017 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11018
11019 ins_encode %{
11020 __ orr(as_Register($dst$$reg),
11021 as_Register($src1$$reg),
11022 as_Register($src2$$reg),
11023 Assembler::ASR,
11024 $src3$$constant & 0x3f);
11025 %}
11026
11027 ins_pipe(ialu_reg_reg_shift);
11028 %}
11029
11030 // This pattern is automatically generated from aarch64_ad.m4
11031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11032 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11033 iRegIorL2I src1, iRegIorL2I src2,
11034 immI src3) %{
11035 match(Set dst (OrI src1 (LShiftI src2 src3)));
11036
11037 ins_cost(1.9 * INSN_COST);
11038 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11039
11040 ins_encode %{
11041 __ orrw(as_Register($dst$$reg),
11042 as_Register($src1$$reg),
11043 as_Register($src2$$reg),
11044 Assembler::LSL,
11045 $src3$$constant & 0x1f);
11046 %}
11047
11048 ins_pipe(ialu_reg_reg_shift);
11049 %}
11050
11051 // This pattern is automatically generated from aarch64_ad.m4
11052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11053 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11054 iRegL src1, iRegL src2,
11055 immI src3) %{
11056 match(Set dst (OrL src1 (LShiftL src2 src3)));
11057
11058 ins_cost(1.9 * INSN_COST);
11059 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11060
11061 ins_encode %{
11062 __ orr(as_Register($dst$$reg),
11063 as_Register($src1$$reg),
11064 as_Register($src2$$reg),
11065 Assembler::LSL,
11066 $src3$$constant & 0x3f);
11067 %}
11068
11069 ins_pipe(ialu_reg_reg_shift);
11070 %}
11071
11072 // This pattern is automatically generated from aarch64_ad.m4
11073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11074 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11075 iRegIorL2I src1, iRegIorL2I src2,
11076 immI src3) %{
11077 match(Set dst (OrI src1 (RotateRight src2 src3)));
11078
11079 ins_cost(1.9 * INSN_COST);
11080 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11081
11082 ins_encode %{
11083 __ orrw(as_Register($dst$$reg),
11084 as_Register($src1$$reg),
11085 as_Register($src2$$reg),
11086 Assembler::ROR,
11087 $src3$$constant & 0x1f);
11088 %}
11089
11090 ins_pipe(ialu_reg_reg_shift);
11091 %}
11092
11093 // This pattern is automatically generated from aarch64_ad.m4
11094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11095 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11096 iRegL src1, iRegL src2,
11097 immI src3) %{
11098 match(Set dst (OrL src1 (RotateRight src2 src3)));
11099
11100 ins_cost(1.9 * INSN_COST);
11101 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11102
11103 ins_encode %{
11104 __ orr(as_Register($dst$$reg),
11105 as_Register($src1$$reg),
11106 as_Register($src2$$reg),
11107 Assembler::ROR,
11108 $src3$$constant & 0x3f);
11109 %}
11110
11111 ins_pipe(ialu_reg_reg_shift);
11112 %}
11113
11114 // This pattern is automatically generated from aarch64_ad.m4
11115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11116 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11117 iRegIorL2I src1, iRegIorL2I src2,
11118 immI src3) %{
11119 match(Set dst (AddI src1 (URShiftI src2 src3)));
11120
11121 ins_cost(1.9 * INSN_COST);
11122 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11123
11124 ins_encode %{
11125 __ addw(as_Register($dst$$reg),
11126 as_Register($src1$$reg),
11127 as_Register($src2$$reg),
11128 Assembler::LSR,
11129 $src3$$constant & 0x1f);
11130 %}
11131
11132 ins_pipe(ialu_reg_reg_shift);
11133 %}
11134
11135 // This pattern is automatically generated from aarch64_ad.m4
11136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11137 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11138 iRegL src1, iRegL src2,
11139 immI src3) %{
11140 match(Set dst (AddL src1 (URShiftL src2 src3)));
11141
11142 ins_cost(1.9 * INSN_COST);
11143 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11144
11145 ins_encode %{
11146 __ add(as_Register($dst$$reg),
11147 as_Register($src1$$reg),
11148 as_Register($src2$$reg),
11149 Assembler::LSR,
11150 $src3$$constant & 0x3f);
11151 %}
11152
11153 ins_pipe(ialu_reg_reg_shift);
11154 %}
11155
11156 // This pattern is automatically generated from aarch64_ad.m4
11157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11158 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11159 iRegIorL2I src1, iRegIorL2I src2,
11160 immI src3) %{
11161 match(Set dst (AddI src1 (RShiftI src2 src3)));
11162
11163 ins_cost(1.9 * INSN_COST);
11164 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11165
11166 ins_encode %{
11167 __ addw(as_Register($dst$$reg),
11168 as_Register($src1$$reg),
11169 as_Register($src2$$reg),
11170 Assembler::ASR,
11171 $src3$$constant & 0x1f);
11172 %}
11173
11174 ins_pipe(ialu_reg_reg_shift);
11175 %}
11176
11177 // This pattern is automatically generated from aarch64_ad.m4
11178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11179 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11180 iRegL src1, iRegL src2,
11181 immI src3) %{
11182 match(Set dst (AddL src1 (RShiftL src2 src3)));
11183
11184 ins_cost(1.9 * INSN_COST);
11185 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11186
11187 ins_encode %{
11188 __ add(as_Register($dst$$reg),
11189 as_Register($src1$$reg),
11190 as_Register($src2$$reg),
11191 Assembler::ASR,
11192 $src3$$constant & 0x3f);
11193 %}
11194
11195 ins_pipe(ialu_reg_reg_shift);
11196 %}
11197
11198 // This pattern is automatically generated from aarch64_ad.m4
11199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11200 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11201 iRegIorL2I src1, iRegIorL2I src2,
11202 immI src3) %{
11203 match(Set dst (AddI src1 (LShiftI src2 src3)));
11204
11205 ins_cost(1.9 * INSN_COST);
11206 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11207
11208 ins_encode %{
11209 __ addw(as_Register($dst$$reg),
11210 as_Register($src1$$reg),
11211 as_Register($src2$$reg),
11212 Assembler::LSL,
11213 $src3$$constant & 0x1f);
11214 %}
11215
11216 ins_pipe(ialu_reg_reg_shift);
11217 %}
11218
11219 // This pattern is automatically generated from aarch64_ad.m4
11220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11221 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11222 iRegL src1, iRegL src2,
11223 immI src3) %{
11224 match(Set dst (AddL src1 (LShiftL src2 src3)));
11225
11226 ins_cost(1.9 * INSN_COST);
11227 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11228
11229 ins_encode %{
11230 __ add(as_Register($dst$$reg),
11231 as_Register($src1$$reg),
11232 as_Register($src2$$reg),
11233 Assembler::LSL,
11234 $src3$$constant & 0x3f);
11235 %}
11236
11237 ins_pipe(ialu_reg_reg_shift);
11238 %}
11239
11240 // This pattern is automatically generated from aarch64_ad.m4
11241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11242 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11243 iRegIorL2I src1, iRegIorL2I src2,
11244 immI src3) %{
11245 match(Set dst (SubI src1 (URShiftI src2 src3)));
11246
11247 ins_cost(1.9 * INSN_COST);
11248 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11249
11250 ins_encode %{
11251 __ subw(as_Register($dst$$reg),
11252 as_Register($src1$$reg),
11253 as_Register($src2$$reg),
11254 Assembler::LSR,
11255 $src3$$constant & 0x1f);
11256 %}
11257
11258 ins_pipe(ialu_reg_reg_shift);
11259 %}
11260
11261 // This pattern is automatically generated from aarch64_ad.m4
11262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11263 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11264 iRegL src1, iRegL src2,
11265 immI src3) %{
11266 match(Set dst (SubL src1 (URShiftL src2 src3)));
11267
11268 ins_cost(1.9 * INSN_COST);
11269 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11270
11271 ins_encode %{
11272 __ sub(as_Register($dst$$reg),
11273 as_Register($src1$$reg),
11274 as_Register($src2$$reg),
11275 Assembler::LSR,
11276 $src3$$constant & 0x3f);
11277 %}
11278
11279 ins_pipe(ialu_reg_reg_shift);
11280 %}
11281
11282 // This pattern is automatically generated from aarch64_ad.m4
11283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11284 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11285 iRegIorL2I src1, iRegIorL2I src2,
11286 immI src3) %{
11287 match(Set dst (SubI src1 (RShiftI src2 src3)));
11288
11289 ins_cost(1.9 * INSN_COST);
11290 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11291
11292 ins_encode %{
11293 __ subw(as_Register($dst$$reg),
11294 as_Register($src1$$reg),
11295 as_Register($src2$$reg),
11296 Assembler::ASR,
11297 $src3$$constant & 0x1f);
11298 %}
11299
11300 ins_pipe(ialu_reg_reg_shift);
11301 %}
11302
11303 // This pattern is automatically generated from aarch64_ad.m4
11304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11305 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11306 iRegL src1, iRegL src2,
11307 immI src3) %{
11308 match(Set dst (SubL src1 (RShiftL src2 src3)));
11309
11310 ins_cost(1.9 * INSN_COST);
11311 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11312
11313 ins_encode %{
11314 __ sub(as_Register($dst$$reg),
11315 as_Register($src1$$reg),
11316 as_Register($src2$$reg),
11317 Assembler::ASR,
11318 $src3$$constant & 0x3f);
11319 %}
11320
11321 ins_pipe(ialu_reg_reg_shift);
11322 %}
11323
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11327 iRegIorL2I src1, iRegIorL2I src2,
11328 immI src3) %{
11329 match(Set dst (SubI src1 (LShiftI src2 src3)));
11330
11331 ins_cost(1.9 * INSN_COST);
11332 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11333
11334 ins_encode %{
11335 __ subw(as_Register($dst$$reg),
11336 as_Register($src1$$reg),
11337 as_Register($src2$$reg),
11338 Assembler::LSL,
11339 $src3$$constant & 0x1f);
11340 %}
11341
11342 ins_pipe(ialu_reg_reg_shift);
11343 %}
11344
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11348 iRegL src1, iRegL src2,
11349 immI src3) %{
11350 match(Set dst (SubL src1 (LShiftL src2 src3)));
11351
11352 ins_cost(1.9 * INSN_COST);
11353 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11354
11355 ins_encode %{
11356 __ sub(as_Register($dst$$reg),
11357 as_Register($src1$$reg),
11358 as_Register($src2$$reg),
11359 Assembler::LSL,
11360 $src3$$constant & 0x3f);
11361 %}
11362
11363 ins_pipe(ialu_reg_reg_shift);
11364 %}
11365
11366 // This pattern is automatically generated from aarch64_ad.m4
11367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11368
11369 // Shift Left followed by Shift Right.
11370 // This idiom is used by the compiler for the i2b bytecode etc.
11371 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11372 %{
11373 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11374 ins_cost(INSN_COST * 2);
11375 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11376 ins_encode %{
11377 int lshift = $lshift_count$$constant & 63;
11378 int rshift = $rshift_count$$constant & 63;
11379 int s = 63 - lshift;
11380 int r = (rshift - lshift) & 63;
11381 __ sbfm(as_Register($dst$$reg),
11382 as_Register($src$$reg),
11383 r, s);
11384 %}
11385
11386 ins_pipe(ialu_reg_shift);
11387 %}
11388
11389 // This pattern is automatically generated from aarch64_ad.m4
11390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11391
11392 // Shift Left followed by Shift Right.
11393 // This idiom is used by the compiler for the i2b bytecode etc.
11394 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11395 %{
11396 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11397 ins_cost(INSN_COST * 2);
11398 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11399 ins_encode %{
11400 int lshift = $lshift_count$$constant & 31;
11401 int rshift = $rshift_count$$constant & 31;
11402 int s = 31 - lshift;
11403 int r = (rshift - lshift) & 31;
11404 __ sbfmw(as_Register($dst$$reg),
11405 as_Register($src$$reg),
11406 r, s);
11407 %}
11408
11409 ins_pipe(ialu_reg_shift);
11410 %}
11411
11412 // This pattern is automatically generated from aarch64_ad.m4
11413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11414
11415 // Shift Left followed by Shift Right.
11416 // This idiom is used by the compiler for the i2b bytecode etc.
11417 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11418 %{
11419 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11420 ins_cost(INSN_COST * 2);
11421 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11422 ins_encode %{
11423 int lshift = $lshift_count$$constant & 63;
11424 int rshift = $rshift_count$$constant & 63;
11425 int s = 63 - lshift;
11426 int r = (rshift - lshift) & 63;
11427 __ ubfm(as_Register($dst$$reg),
11428 as_Register($src$$reg),
11429 r, s);
11430 %}
11431
11432 ins_pipe(ialu_reg_shift);
11433 %}
11434
11435 // This pattern is automatically generated from aarch64_ad.m4
11436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11437
11438 // Shift Left followed by Shift Right.
11439 // This idiom is used by the compiler for the i2b bytecode etc.
11440 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11441 %{
11442 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11443 ins_cost(INSN_COST * 2);
11444 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11445 ins_encode %{
11446 int lshift = $lshift_count$$constant & 31;
11447 int rshift = $rshift_count$$constant & 31;
11448 int s = 31 - lshift;
11449 int r = (rshift - lshift) & 31;
11450 __ ubfmw(as_Register($dst$$reg),
11451 as_Register($src$$reg),
11452 r, s);
11453 %}
11454
11455 ins_pipe(ialu_reg_shift);
11456 %}
11457
11458 // Bitfield extract with shift & mask
11459
11460 // This pattern is automatically generated from aarch64_ad.m4
11461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11462 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11463 %{
11464 match(Set dst (AndI (URShiftI src rshift) mask));
11465 // Make sure we are not going to exceed what ubfxw can do.
11466 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11467
11468 ins_cost(INSN_COST);
11469 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11470 ins_encode %{
11471 int rshift = $rshift$$constant & 31;
11472 intptr_t mask = $mask$$constant;
11473 int width = exact_log2(mask+1);
11474 __ ubfxw(as_Register($dst$$reg),
11475 as_Register($src$$reg), rshift, width);
11476 %}
11477 ins_pipe(ialu_reg_shift);
11478 %}
11479
11480 // This pattern is automatically generated from aarch64_ad.m4
11481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11482 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11483 %{
11484 match(Set dst (AndL (URShiftL src rshift) mask));
11485 // Make sure we are not going to exceed what ubfx can do.
11486 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11487
11488 ins_cost(INSN_COST);
11489 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11490 ins_encode %{
11491 int rshift = $rshift$$constant & 63;
11492 intptr_t mask = $mask$$constant;
11493 int width = exact_log2_long(mask+1);
11494 __ ubfx(as_Register($dst$$reg),
11495 as_Register($src$$reg), rshift, width);
11496 %}
11497 ins_pipe(ialu_reg_shift);
11498 %}
11499
11500
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503
11504 // We can use ubfx when extending an And with a mask when we know mask
11505 // is positive. We know that because immI_bitmask guarantees it.
11506 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11507 %{
11508 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11509 // Make sure we are not going to exceed what ubfxw can do.
11510 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11511
11512 ins_cost(INSN_COST * 2);
11513 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11514 ins_encode %{
11515 int rshift = $rshift$$constant & 31;
11516 intptr_t mask = $mask$$constant;
11517 int width = exact_log2(mask+1);
11518 __ ubfx(as_Register($dst$$reg),
11519 as_Register($src$$reg), rshift, width);
11520 %}
11521 ins_pipe(ialu_reg_shift);
11522 %}
11523
11524
11525 // This pattern is automatically generated from aarch64_ad.m4
11526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11527
11528 // We can use ubfiz when masking by a positive number and then left shifting the result.
11529 // We know that the mask is positive because immI_bitmask guarantees it.
11530 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11531 %{
11532 match(Set dst (LShiftI (AndI src mask) lshift));
11533 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11534
11535 ins_cost(INSN_COST);
11536 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11537 ins_encode %{
11538 int lshift = $lshift$$constant & 31;
11539 intptr_t mask = $mask$$constant;
11540 int width = exact_log2(mask+1);
11541 __ ubfizw(as_Register($dst$$reg),
11542 as_Register($src$$reg), lshift, width);
11543 %}
11544 ins_pipe(ialu_reg_shift);
11545 %}
11546
11547 // This pattern is automatically generated from aarch64_ad.m4
11548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11549
11550 // We can use ubfiz when masking by a positive number and then left shifting the result.
11551 // We know that the mask is positive because immL_bitmask guarantees it.
11552 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11553 %{
11554 match(Set dst (LShiftL (AndL src mask) lshift));
11555 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11556
11557 ins_cost(INSN_COST);
11558 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11559 ins_encode %{
11560 int lshift = $lshift$$constant & 63;
11561 intptr_t mask = $mask$$constant;
11562 int width = exact_log2_long(mask+1);
11563 __ ubfiz(as_Register($dst$$reg),
11564 as_Register($src$$reg), lshift, width);
11565 %}
11566 ins_pipe(ialu_reg_shift);
11567 %}
11568
11569 // This pattern is automatically generated from aarch64_ad.m4
11570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11571
11572 // We can use ubfiz when masking by a positive number and then left shifting the result.
11573 // We know that the mask is positive because immI_bitmask guarantees it.
11574 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11575 %{
11576 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11577 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11578
11579 ins_cost(INSN_COST);
11580 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11581 ins_encode %{
11582 int lshift = $lshift$$constant & 31;
11583 intptr_t mask = $mask$$constant;
11584 int width = exact_log2(mask+1);
11585 __ ubfizw(as_Register($dst$$reg),
11586 as_Register($src$$reg), lshift, width);
11587 %}
11588 ins_pipe(ialu_reg_shift);
11589 %}
11590
11591 // This pattern is automatically generated from aarch64_ad.m4
11592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11593
11594 // We can use ubfiz when masking by a positive number and then left shifting the result.
11595 // We know that the mask is positive because immL_bitmask guarantees it.
11596 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11597 %{
11598 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11599 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11600
11601 ins_cost(INSN_COST);
11602 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11603 ins_encode %{
11604 int lshift = $lshift$$constant & 63;
11605 intptr_t mask = $mask$$constant;
11606 int width = exact_log2_long(mask+1);
11607 __ ubfiz(as_Register($dst$$reg),
11608 as_Register($src$$reg), lshift, width);
11609 %}
11610 ins_pipe(ialu_reg_shift);
11611 %}
11612
11613
11614 // This pattern is automatically generated from aarch64_ad.m4
11615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11616
11617 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11618 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11619 %{
11620 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11621 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11622
11623 ins_cost(INSN_COST);
11624 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11625 ins_encode %{
11626 int lshift = $lshift$$constant & 63;
11627 intptr_t mask = $mask$$constant;
11628 int width = exact_log2(mask+1);
11629 __ ubfiz(as_Register($dst$$reg),
11630 as_Register($src$$reg), lshift, width);
11631 %}
11632 ins_pipe(ialu_reg_shift);
11633 %}
11634
11635 // This pattern is automatically generated from aarch64_ad.m4
11636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11637
11638 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11639 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11640 %{
11641 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11642 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11643
11644 ins_cost(INSN_COST);
11645 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11646 ins_encode %{
11647 int lshift = $lshift$$constant & 31;
11648 intptr_t mask = $mask$$constant;
11649 int width = exact_log2(mask+1);
11650 __ ubfiz(as_Register($dst$$reg),
11651 as_Register($src$$reg), lshift, width);
11652 %}
11653 ins_pipe(ialu_reg_shift);
11654 %}
11655
11656 // This pattern is automatically generated from aarch64_ad.m4
11657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11658
11659 // Can skip int2long conversions after AND with small bitmask
11660 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11661 %{
11662 match(Set dst (ConvI2L (AndI src msk)));
11663 ins_cost(INSN_COST);
11664 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11665 ins_encode %{
11666 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11667 %}
11668 ins_pipe(ialu_reg_shift);
11669 %}
11670
11671
11672 // Rotations
11673
11674 // This pattern is automatically generated from aarch64_ad.m4
11675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11676 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11677 %{
11678 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11679 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11680
11681 ins_cost(INSN_COST);
11682 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11683
11684 ins_encode %{
11685 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11686 $rshift$$constant & 63);
11687 %}
11688 ins_pipe(ialu_reg_reg_extr);
11689 %}
11690
11691
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11695 %{
11696 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11697 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11698
11699 ins_cost(INSN_COST);
11700 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11701
11702 ins_encode %{
11703 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11704 $rshift$$constant & 31);
11705 %}
11706 ins_pipe(ialu_reg_reg_extr);
11707 %}
11708
11709
11710 // This pattern is automatically generated from aarch64_ad.m4
11711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11712 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11713 %{
11714 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11715 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11716
11717 ins_cost(INSN_COST);
11718 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11719
11720 ins_encode %{
11721 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11722 $rshift$$constant & 63);
11723 %}
11724 ins_pipe(ialu_reg_reg_extr);
11725 %}
11726
11727
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11731 %{
11732 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11733 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11734
11735 ins_cost(INSN_COST);
11736 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11737
11738 ins_encode %{
11739 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11740 $rshift$$constant & 31);
11741 %}
11742 ins_pipe(ialu_reg_reg_extr);
11743 %}
11744
11745 // This pattern is automatically generated from aarch64_ad.m4
11746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11747 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11748 %{
11749 match(Set dst (RotateRight src shift));
11750
11751 ins_cost(INSN_COST);
11752 format %{ "ror $dst, $src, $shift" %}
11753
11754 ins_encode %{
11755 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11756 $shift$$constant & 0x1f);
11757 %}
11758 ins_pipe(ialu_reg_reg_vshift);
11759 %}
11760
11761 // This pattern is automatically generated from aarch64_ad.m4
11762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11763 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11764 %{
11765 match(Set dst (RotateRight src shift));
11766
11767 ins_cost(INSN_COST);
11768 format %{ "ror $dst, $src, $shift" %}
11769
11770 ins_encode %{
11771 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11772 $shift$$constant & 0x3f);
11773 %}
11774 ins_pipe(ialu_reg_reg_vshift);
11775 %}
11776
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11780 %{
11781 match(Set dst (RotateRight src shift));
11782
11783 ins_cost(INSN_COST);
11784 format %{ "ror $dst, $src, $shift" %}
11785
11786 ins_encode %{
11787 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11788 %}
11789 ins_pipe(ialu_reg_reg_vshift);
11790 %}
11791
11792 // This pattern is automatically generated from aarch64_ad.m4
11793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11794 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11795 %{
11796 match(Set dst (RotateRight src shift));
11797
11798 ins_cost(INSN_COST);
11799 format %{ "ror $dst, $src, $shift" %}
11800
11801 ins_encode %{
11802 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11803 %}
11804 ins_pipe(ialu_reg_reg_vshift);
11805 %}
11806
11807 // This pattern is automatically generated from aarch64_ad.m4
11808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11809 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11810 %{
11811 match(Set dst (RotateLeft src shift));
11812
11813 ins_cost(INSN_COST);
11814 format %{ "rol $dst, $src, $shift" %}
11815
11816 ins_encode %{
11817 __ subw(rscratch1, zr, as_Register($shift$$reg));
11818 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11819 %}
11820 ins_pipe(ialu_reg_reg_vshift);
11821 %}
11822
11823 // This pattern is automatically generated from aarch64_ad.m4
11824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11825 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11826 %{
11827 match(Set dst (RotateLeft src shift));
11828
11829 ins_cost(INSN_COST);
11830 format %{ "rol $dst, $src, $shift" %}
11831
11832 ins_encode %{
11833 __ subw(rscratch1, zr, as_Register($shift$$reg));
11834 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11835 %}
11836 ins_pipe(ialu_reg_reg_vshift);
11837 %}
11838
11839
11840 // Add/subtract (extended)
11841
11842 // This pattern is automatically generated from aarch64_ad.m4
11843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11844 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11845 %{
11846 match(Set dst (AddL src1 (ConvI2L src2)));
11847 ins_cost(INSN_COST);
11848 format %{ "add $dst, $src1, $src2, sxtw" %}
11849
11850 ins_encode %{
11851 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11852 as_Register($src2$$reg), ext::sxtw);
11853 %}
11854 ins_pipe(ialu_reg_reg);
11855 %}
11856
11857 // This pattern is automatically generated from aarch64_ad.m4
11858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11859 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11860 %{
11861 match(Set dst (SubL src1 (ConvI2L src2)));
11862 ins_cost(INSN_COST);
11863 format %{ "sub $dst, $src1, $src2, sxtw" %}
11864
11865 ins_encode %{
11866 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11867 as_Register($src2$$reg), ext::sxtw);
11868 %}
11869 ins_pipe(ialu_reg_reg);
11870 %}
11871
11872 // This pattern is automatically generated from aarch64_ad.m4
11873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11874 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11875 %{
11876 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11877 ins_cost(INSN_COST);
11878 format %{ "add $dst, $src1, $src2, sxth" %}
11879
11880 ins_encode %{
11881 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11882 as_Register($src2$$reg), ext::sxth);
11883 %}
11884 ins_pipe(ialu_reg_reg);
11885 %}
11886
11887 // This pattern is automatically generated from aarch64_ad.m4
11888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11889 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11890 %{
11891 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11892 ins_cost(INSN_COST);
11893 format %{ "add $dst, $src1, $src2, sxtb" %}
11894
11895 ins_encode %{
11896 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11897 as_Register($src2$$reg), ext::sxtb);
11898 %}
11899 ins_pipe(ialu_reg_reg);
11900 %}
11901
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11905 %{
11906 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11907 ins_cost(INSN_COST);
11908 format %{ "add $dst, $src1, $src2, uxtb" %}
11909
11910 ins_encode %{
11911 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11912 as_Register($src2$$reg), ext::uxtb);
11913 %}
11914 ins_pipe(ialu_reg_reg);
11915 %}
11916
11917 // This pattern is automatically generated from aarch64_ad.m4
11918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11919 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11920 %{
11921 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11922 ins_cost(INSN_COST);
11923 format %{ "add $dst, $src1, $src2, sxth" %}
11924
11925 ins_encode %{
11926 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11927 as_Register($src2$$reg), ext::sxth);
11928 %}
11929 ins_pipe(ialu_reg_reg);
11930 %}
11931
11932 // This pattern is automatically generated from aarch64_ad.m4
11933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11934 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11935 %{
11936 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11937 ins_cost(INSN_COST);
11938 format %{ "add $dst, $src1, $src2, sxtw" %}
11939
11940 ins_encode %{
11941 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11942 as_Register($src2$$reg), ext::sxtw);
11943 %}
11944 ins_pipe(ialu_reg_reg);
11945 %}
11946
11947 // This pattern is automatically generated from aarch64_ad.m4
11948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11949 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11950 %{
11951 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11952 ins_cost(INSN_COST);
11953 format %{ "add $dst, $src1, $src2, sxtb" %}
11954
11955 ins_encode %{
11956 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11957 as_Register($src2$$reg), ext::sxtb);
11958 %}
11959 ins_pipe(ialu_reg_reg);
11960 %}
11961
11962 // This pattern is automatically generated from aarch64_ad.m4
11963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11964 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11965 %{
11966 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11967 ins_cost(INSN_COST);
11968 format %{ "add $dst, $src1, $src2, uxtb" %}
11969
11970 ins_encode %{
11971 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11972 as_Register($src2$$reg), ext::uxtb);
11973 %}
11974 ins_pipe(ialu_reg_reg);
11975 %}
11976
11977 // This pattern is automatically generated from aarch64_ad.m4
11978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11979 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11980 %{
11981 match(Set dst (AddI src1 (AndI src2 mask)));
11982 ins_cost(INSN_COST);
11983 format %{ "addw $dst, $src1, $src2, uxtb" %}
11984
11985 ins_encode %{
11986 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11987 as_Register($src2$$reg), ext::uxtb);
11988 %}
11989 ins_pipe(ialu_reg_reg);
11990 %}
11991
11992 // This pattern is automatically generated from aarch64_ad.m4
11993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11994 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11995 %{
11996 match(Set dst (AddI src1 (AndI src2 mask)));
11997 ins_cost(INSN_COST);
11998 format %{ "addw $dst, $src1, $src2, uxth" %}
11999
12000 ins_encode %{
12001 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12002 as_Register($src2$$reg), ext::uxth);
12003 %}
12004 ins_pipe(ialu_reg_reg);
12005 %}
12006
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12010 %{
12011 match(Set dst (AddL src1 (AndL src2 mask)));
12012 ins_cost(INSN_COST);
12013 format %{ "add $dst, $src1, $src2, uxtb" %}
12014
12015 ins_encode %{
12016 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12017 as_Register($src2$$reg), ext::uxtb);
12018 %}
12019 ins_pipe(ialu_reg_reg);
12020 %}
12021
12022 // This pattern is automatically generated from aarch64_ad.m4
12023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12024 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12025 %{
12026 match(Set dst (AddL src1 (AndL src2 mask)));
12027 ins_cost(INSN_COST);
12028 format %{ "add $dst, $src1, $src2, uxth" %}
12029
12030 ins_encode %{
12031 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12032 as_Register($src2$$reg), ext::uxth);
12033 %}
12034 ins_pipe(ialu_reg_reg);
12035 %}
12036
12037 // This pattern is automatically generated from aarch64_ad.m4
12038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12039 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12040 %{
12041 match(Set dst (AddL src1 (AndL src2 mask)));
12042 ins_cost(INSN_COST);
12043 format %{ "add $dst, $src1, $src2, uxtw" %}
12044
12045 ins_encode %{
12046 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12047 as_Register($src2$$reg), ext::uxtw);
12048 %}
12049 ins_pipe(ialu_reg_reg);
12050 %}
12051
12052 // This pattern is automatically generated from aarch64_ad.m4
12053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12054 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12055 %{
12056 match(Set dst (SubI src1 (AndI src2 mask)));
12057 ins_cost(INSN_COST);
12058 format %{ "subw $dst, $src1, $src2, uxtb" %}
12059
12060 ins_encode %{
12061 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12062 as_Register($src2$$reg), ext::uxtb);
12063 %}
12064 ins_pipe(ialu_reg_reg);
12065 %}
12066
12067 // This pattern is automatically generated from aarch64_ad.m4
12068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12069 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12070 %{
12071 match(Set dst (SubI src1 (AndI src2 mask)));
12072 ins_cost(INSN_COST);
12073 format %{ "subw $dst, $src1, $src2, uxth" %}
12074
12075 ins_encode %{
12076 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12077 as_Register($src2$$reg), ext::uxth);
12078 %}
12079 ins_pipe(ialu_reg_reg);
12080 %}
12081
12082 // This pattern is automatically generated from aarch64_ad.m4
12083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12084 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12085 %{
12086 match(Set dst (SubL src1 (AndL src2 mask)));
12087 ins_cost(INSN_COST);
12088 format %{ "sub $dst, $src1, $src2, uxtb" %}
12089
12090 ins_encode %{
12091 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12092 as_Register($src2$$reg), ext::uxtb);
12093 %}
12094 ins_pipe(ialu_reg_reg);
12095 %}
12096
12097 // This pattern is automatically generated from aarch64_ad.m4
12098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12099 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12100 %{
12101 match(Set dst (SubL src1 (AndL src2 mask)));
12102 ins_cost(INSN_COST);
12103 format %{ "sub $dst, $src1, $src2, uxth" %}
12104
12105 ins_encode %{
12106 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12107 as_Register($src2$$reg), ext::uxth);
12108 %}
12109 ins_pipe(ialu_reg_reg);
12110 %}
12111
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12115 %{
12116 match(Set dst (SubL src1 (AndL src2 mask)));
12117 ins_cost(INSN_COST);
12118 format %{ "sub $dst, $src1, $src2, uxtw" %}
12119
12120 ins_encode %{
12121 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12122 as_Register($src2$$reg), ext::uxtw);
12123 %}
12124 ins_pipe(ialu_reg_reg);
12125 %}
12126
12127
12128 // This pattern is automatically generated from aarch64_ad.m4
12129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12130 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12131 %{
12132 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12133 ins_cost(1.9 * INSN_COST);
12134 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12135
12136 ins_encode %{
12137 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12138 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12139 %}
12140 ins_pipe(ialu_reg_reg_shift);
12141 %}
12142
12143 // This pattern is automatically generated from aarch64_ad.m4
12144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12145 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12146 %{
12147 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12148 ins_cost(1.9 * INSN_COST);
12149 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12150
12151 ins_encode %{
12152 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12153 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12154 %}
12155 ins_pipe(ialu_reg_reg_shift);
12156 %}
12157
12158 // This pattern is automatically generated from aarch64_ad.m4
12159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12160 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12161 %{
12162 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12163 ins_cost(1.9 * INSN_COST);
12164 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12165
12166 ins_encode %{
12167 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12168 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12169 %}
12170 ins_pipe(ialu_reg_reg_shift);
12171 %}
12172
12173 // This pattern is automatically generated from aarch64_ad.m4
12174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12175 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12176 %{
12177 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12178 ins_cost(1.9 * INSN_COST);
12179 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12180
12181 ins_encode %{
12182 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12183 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12184 %}
12185 ins_pipe(ialu_reg_reg_shift);
12186 %}
12187
12188 // This pattern is automatically generated from aarch64_ad.m4
12189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12190 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12191 %{
12192 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12193 ins_cost(1.9 * INSN_COST);
12194 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12195
12196 ins_encode %{
12197 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12198 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12199 %}
12200 ins_pipe(ialu_reg_reg_shift);
12201 %}
12202
12203 // This pattern is automatically generated from aarch64_ad.m4
12204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12205 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12206 %{
12207 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12208 ins_cost(1.9 * INSN_COST);
12209 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12210
12211 ins_encode %{
12212 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12213 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12214 %}
12215 ins_pipe(ialu_reg_reg_shift);
12216 %}
12217
12218 // This pattern is automatically generated from aarch64_ad.m4
12219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12220 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12221 %{
12222 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12223 ins_cost(1.9 * INSN_COST);
12224 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12225
12226 ins_encode %{
12227 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12228 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12229 %}
12230 ins_pipe(ialu_reg_reg_shift);
12231 %}
12232
12233 // This pattern is automatically generated from aarch64_ad.m4
12234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12235 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12236 %{
12237 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12238 ins_cost(1.9 * INSN_COST);
12239 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12240
12241 ins_encode %{
12242 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12243 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12244 %}
12245 ins_pipe(ialu_reg_reg_shift);
12246 %}
12247
12248 // This pattern is automatically generated from aarch64_ad.m4
12249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12250 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12251 %{
12252 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12253 ins_cost(1.9 * INSN_COST);
12254 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12255
12256 ins_encode %{
12257 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12258 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12259 %}
12260 ins_pipe(ialu_reg_reg_shift);
12261 %}
12262
12263 // This pattern is automatically generated from aarch64_ad.m4
12264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12265 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12266 %{
12267 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12268 ins_cost(1.9 * INSN_COST);
12269 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12270
12271 ins_encode %{
12272 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12273 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12274 %}
12275 ins_pipe(ialu_reg_reg_shift);
12276 %}
12277
12278 // This pattern is automatically generated from aarch64_ad.m4
12279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12280 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12281 %{
12282 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12283 ins_cost(1.9 * INSN_COST);
12284 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12285
12286 ins_encode %{
12287 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12288 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12289 %}
12290 ins_pipe(ialu_reg_reg_shift);
12291 %}
12292
12293 // This pattern is automatically generated from aarch64_ad.m4
12294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12295 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12296 %{
12297 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12298 ins_cost(1.9 * INSN_COST);
12299 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12300
12301 ins_encode %{
12302 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12303 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12304 %}
12305 ins_pipe(ialu_reg_reg_shift);
12306 %}
12307
12308 // This pattern is automatically generated from aarch64_ad.m4
12309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12310 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12311 %{
12312 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12313 ins_cost(1.9 * INSN_COST);
12314 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12315
12316 ins_encode %{
12317 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12318 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12319 %}
12320 ins_pipe(ialu_reg_reg_shift);
12321 %}
12322
12323 // This pattern is automatically generated from aarch64_ad.m4
12324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12325 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12326 %{
12327 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12328 ins_cost(1.9 * INSN_COST);
12329 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12330
12331 ins_encode %{
12332 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12333 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12334 %}
12335 ins_pipe(ialu_reg_reg_shift);
12336 %}
12337
12338 // This pattern is automatically generated from aarch64_ad.m4
12339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12340 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12341 %{
12342 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12343 ins_cost(1.9 * INSN_COST);
12344 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12345
12346 ins_encode %{
12347 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12348 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12349 %}
12350 ins_pipe(ialu_reg_reg_shift);
12351 %}
12352
12353 // This pattern is automatically generated from aarch64_ad.m4
12354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12355 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12356 %{
12357 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12358 ins_cost(1.9 * INSN_COST);
12359 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12360
12361 ins_encode %{
12362 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12363 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12364 %}
12365 ins_pipe(ialu_reg_reg_shift);
12366 %}
12367
12368 // This pattern is automatically generated from aarch64_ad.m4
12369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12370 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12371 %{
12372 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12373 ins_cost(1.9 * INSN_COST);
12374 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12375
12376 ins_encode %{
12377 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12378 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12379 %}
12380 ins_pipe(ialu_reg_reg_shift);
12381 %}
12382
12383 // This pattern is automatically generated from aarch64_ad.m4
12384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12385 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12386 %{
12387 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12388 ins_cost(1.9 * INSN_COST);
12389 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12390
12391 ins_encode %{
12392 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12393 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12394 %}
12395 ins_pipe(ialu_reg_reg_shift);
12396 %}
12397
12398 // This pattern is automatically generated from aarch64_ad.m4
12399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12400 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12401 %{
12402 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12403 ins_cost(1.9 * INSN_COST);
12404 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12405
12406 ins_encode %{
12407 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12408 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12409 %}
12410 ins_pipe(ialu_reg_reg_shift);
12411 %}
12412
12413 // This pattern is automatically generated from aarch64_ad.m4
12414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12415 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12416 %{
12417 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12418 ins_cost(1.9 * INSN_COST);
12419 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12420
12421 ins_encode %{
12422 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12423 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12424 %}
12425 ins_pipe(ialu_reg_reg_shift);
12426 %}
12427
12428 // This pattern is automatically generated from aarch64_ad.m4
12429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12430 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12431 %{
12432 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12433 ins_cost(1.9 * INSN_COST);
12434 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12435
12436 ins_encode %{
12437 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12438 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12439 %}
12440 ins_pipe(ialu_reg_reg_shift);
12441 %}
12442
12443 // This pattern is automatically generated from aarch64_ad.m4
12444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12445 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12446 %{
12447 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12448 ins_cost(1.9 * INSN_COST);
12449 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12450
12451 ins_encode %{
12452 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12453 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12454 %}
12455 ins_pipe(ialu_reg_reg_shift);
12456 %}
12457
12458 // This pattern is automatically generated from aarch64_ad.m4
12459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12460 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12461 %{
12462 effect(DEF dst, USE src1, USE src2, USE cr);
12463 ins_cost(INSN_COST * 2);
12464 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12465
12466 ins_encode %{
12467 __ cselw($dst$$Register,
12468 $src1$$Register,
12469 $src2$$Register,
12470 Assembler::LT);
12471 %}
12472 ins_pipe(icond_reg_reg);
12473 %}
12474
12475 // This pattern is automatically generated from aarch64_ad.m4
12476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12477 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12478 %{
12479 effect(DEF dst, USE src1, USE src2, USE cr);
12480 ins_cost(INSN_COST * 2);
12481 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12482
12483 ins_encode %{
12484 __ cselw($dst$$Register,
12485 $src1$$Register,
12486 $src2$$Register,
12487 Assembler::GT);
12488 %}
12489 ins_pipe(icond_reg_reg);
12490 %}
12491
12492 // This pattern is automatically generated from aarch64_ad.m4
12493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12494 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12495 %{
12496 effect(DEF dst, USE src1, USE cr);
12497 ins_cost(INSN_COST * 2);
12498 format %{ "cselw $dst, $src1, zr lt\t" %}
12499
12500 ins_encode %{
12501 __ cselw($dst$$Register,
12502 $src1$$Register,
12503 zr,
12504 Assembler::LT);
12505 %}
12506 ins_pipe(icond_reg);
12507 %}
12508
12509 // This pattern is automatically generated from aarch64_ad.m4
12510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12511 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12512 %{
12513 effect(DEF dst, USE src1, USE cr);
12514 ins_cost(INSN_COST * 2);
12515 format %{ "cselw $dst, $src1, zr gt\t" %}
12516
12517 ins_encode %{
12518 __ cselw($dst$$Register,
12519 $src1$$Register,
12520 zr,
12521 Assembler::GT);
12522 %}
12523 ins_pipe(icond_reg);
12524 %}
12525
12526 // This pattern is automatically generated from aarch64_ad.m4
12527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12528 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12529 %{
12530 effect(DEF dst, USE src1, USE cr);
12531 ins_cost(INSN_COST * 2);
12532 format %{ "csincw $dst, $src1, zr le\t" %}
12533
12534 ins_encode %{
12535 __ csincw($dst$$Register,
12536 $src1$$Register,
12537 zr,
12538 Assembler::LE);
12539 %}
12540 ins_pipe(icond_reg);
12541 %}
12542
12543 // This pattern is automatically generated from aarch64_ad.m4
12544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12545 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12546 %{
12547 effect(DEF dst, USE src1, USE cr);
12548 ins_cost(INSN_COST * 2);
12549 format %{ "csincw $dst, $src1, zr gt\t" %}
12550
12551 ins_encode %{
12552 __ csincw($dst$$Register,
12553 $src1$$Register,
12554 zr,
12555 Assembler::GT);
12556 %}
12557 ins_pipe(icond_reg);
12558 %}
12559
12560 // This pattern is automatically generated from aarch64_ad.m4
12561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12562 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12563 %{
12564 effect(DEF dst, USE src1, USE cr);
12565 ins_cost(INSN_COST * 2);
12566 format %{ "csinvw $dst, $src1, zr lt\t" %}
12567
12568 ins_encode %{
12569 __ csinvw($dst$$Register,
12570 $src1$$Register,
12571 zr,
12572 Assembler::LT);
12573 %}
12574 ins_pipe(icond_reg);
12575 %}
12576
12577 // This pattern is automatically generated from aarch64_ad.m4
12578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12579 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12580 %{
12581 effect(DEF dst, USE src1, USE cr);
12582 ins_cost(INSN_COST * 2);
12583 format %{ "csinvw $dst, $src1, zr ge\t" %}
12584
12585 ins_encode %{
12586 __ csinvw($dst$$Register,
12587 $src1$$Register,
12588 zr,
12589 Assembler::GE);
12590 %}
12591 ins_pipe(icond_reg);
12592 %}
12593
12594 // This pattern is automatically generated from aarch64_ad.m4
12595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12596 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12597 %{
12598 match(Set dst (MinI src imm));
12599 ins_cost(INSN_COST * 3);
12600 expand %{
12601 rFlagsReg cr;
12602 compI_reg_imm0(cr, src);
12603 cmovI_reg_imm0_lt(dst, src, cr);
12604 %}
12605 %}
12606
12607 // This pattern is automatically generated from aarch64_ad.m4
12608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12609 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12610 %{
12611 match(Set dst (MinI imm src));
12612 ins_cost(INSN_COST * 3);
12613 expand %{
12614 rFlagsReg cr;
12615 compI_reg_imm0(cr, src);
12616 cmovI_reg_imm0_lt(dst, src, cr);
12617 %}
12618 %}
12619
12620 // This pattern is automatically generated from aarch64_ad.m4
12621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12622 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12623 %{
12624 match(Set dst (MinI src imm));
12625 ins_cost(INSN_COST * 3);
12626 expand %{
12627 rFlagsReg cr;
12628 compI_reg_imm0(cr, src);
12629 cmovI_reg_imm1_le(dst, src, cr);
12630 %}
12631 %}
12632
12633 // This pattern is automatically generated from aarch64_ad.m4
12634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12635 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12636 %{
12637 match(Set dst (MinI imm src));
12638 ins_cost(INSN_COST * 3);
12639 expand %{
12640 rFlagsReg cr;
12641 compI_reg_imm0(cr, src);
12642 cmovI_reg_imm1_le(dst, src, cr);
12643 %}
12644 %}
12645
12646 // This pattern is automatically generated from aarch64_ad.m4
12647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12648 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12649 %{
12650 match(Set dst (MinI src imm));
12651 ins_cost(INSN_COST * 3);
12652 expand %{
12653 rFlagsReg cr;
12654 compI_reg_imm0(cr, src);
12655 cmovI_reg_immM1_lt(dst, src, cr);
12656 %}
12657 %}
12658
12659 // This pattern is automatically generated from aarch64_ad.m4
12660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12661 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12662 %{
12663 match(Set dst (MinI imm src));
12664 ins_cost(INSN_COST * 3);
12665 expand %{
12666 rFlagsReg cr;
12667 compI_reg_imm0(cr, src);
12668 cmovI_reg_immM1_lt(dst, src, cr);
12669 %}
12670 %}
12671
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12675 %{
12676 match(Set dst (MaxI src imm));
12677 ins_cost(INSN_COST * 3);
12678 expand %{
12679 rFlagsReg cr;
12680 compI_reg_imm0(cr, src);
12681 cmovI_reg_imm0_gt(dst, src, cr);
12682 %}
12683 %}
12684
12685 // This pattern is automatically generated from aarch64_ad.m4
12686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12687 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12688 %{
12689 match(Set dst (MaxI imm src));
12690 ins_cost(INSN_COST * 3);
12691 expand %{
12692 rFlagsReg cr;
12693 compI_reg_imm0(cr, src);
12694 cmovI_reg_imm0_gt(dst, src, cr);
12695 %}
12696 %}
12697
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12701 %{
12702 match(Set dst (MaxI src imm));
12703 ins_cost(INSN_COST * 3);
12704 expand %{
12705 rFlagsReg cr;
12706 compI_reg_imm0(cr, src);
12707 cmovI_reg_imm1_gt(dst, src, cr);
12708 %}
12709 %}
12710
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12714 %{
12715 match(Set dst (MaxI imm src));
12716 ins_cost(INSN_COST * 3);
12717 expand %{
12718 rFlagsReg cr;
12719 compI_reg_imm0(cr, src);
12720 cmovI_reg_imm1_gt(dst, src, cr);
12721 %}
12722 %}
12723
12724 // This pattern is automatically generated from aarch64_ad.m4
12725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12726 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12727 %{
12728 match(Set dst (MaxI src imm));
12729 ins_cost(INSN_COST * 3);
12730 expand %{
12731 rFlagsReg cr;
12732 compI_reg_imm0(cr, src);
12733 cmovI_reg_immM1_ge(dst, src, cr);
12734 %}
12735 %}
12736
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12740 %{
12741 match(Set dst (MaxI imm src));
12742 ins_cost(INSN_COST * 3);
12743 expand %{
12744 rFlagsReg cr;
12745 compI_reg_imm0(cr, src);
12746 cmovI_reg_immM1_ge(dst, src, cr);
12747 %}
12748 %}
12749
12750 // This pattern is automatically generated from aarch64_ad.m4
12751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12752 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12753 %{
12754 match(Set dst (ReverseI src));
12755 ins_cost(INSN_COST);
12756 format %{ "rbitw $dst, $src" %}
12757 ins_encode %{
12758 __ rbitw($dst$$Register, $src$$Register);
12759 %}
12760 ins_pipe(ialu_reg);
12761 %}
12762
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12766 %{
12767 match(Set dst (ReverseL src));
12768 ins_cost(INSN_COST);
12769 format %{ "rbit $dst, $src" %}
12770 ins_encode %{
12771 __ rbit($dst$$Register, $src$$Register);
12772 %}
12773 ins_pipe(ialu_reg);
12774 %}
12775
12776
12777 // END This section of the file is automatically generated. Do not edit --------------
12778
12779
12780 // ============================================================================
12781 // Floating Point Arithmetic Instructions
12782
12783 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12784 match(Set dst (AddHF src1 src2));
12785 format %{ "faddh $dst, $src1, $src2" %}
12786 ins_encode %{
12787 __ faddh($dst$$FloatRegister,
12788 $src1$$FloatRegister,
12789 $src2$$FloatRegister);
12790 %}
12791 ins_pipe(fp_dop_reg_reg_s);
12792 %}
12793
12794 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12795 match(Set dst (AddF src1 src2));
12796
12797 ins_cost(INSN_COST * 5);
12798 format %{ "fadds $dst, $src1, $src2" %}
12799
12800 ins_encode %{
12801 __ fadds(as_FloatRegister($dst$$reg),
12802 as_FloatRegister($src1$$reg),
12803 as_FloatRegister($src2$$reg));
12804 %}
12805
12806 ins_pipe(fp_dop_reg_reg_s);
12807 %}
12808
12809 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12810 match(Set dst (AddD src1 src2));
12811
12812 ins_cost(INSN_COST * 5);
12813 format %{ "faddd $dst, $src1, $src2" %}
12814
12815 ins_encode %{
12816 __ faddd(as_FloatRegister($dst$$reg),
12817 as_FloatRegister($src1$$reg),
12818 as_FloatRegister($src2$$reg));
12819 %}
12820
12821 ins_pipe(fp_dop_reg_reg_d);
12822 %}
12823
12824 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12825 match(Set dst (SubHF src1 src2));
12826 format %{ "fsubh $dst, $src1, $src2" %}
12827 ins_encode %{
12828 __ fsubh($dst$$FloatRegister,
12829 $src1$$FloatRegister,
12830 $src2$$FloatRegister);
12831 %}
12832 ins_pipe(fp_dop_reg_reg_s);
12833 %}
12834
12835 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12836 match(Set dst (SubF src1 src2));
12837
12838 ins_cost(INSN_COST * 5);
12839 format %{ "fsubs $dst, $src1, $src2" %}
12840
12841 ins_encode %{
12842 __ fsubs(as_FloatRegister($dst$$reg),
12843 as_FloatRegister($src1$$reg),
12844 as_FloatRegister($src2$$reg));
12845 %}
12846
12847 ins_pipe(fp_dop_reg_reg_s);
12848 %}
12849
12850 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12851 match(Set dst (SubD src1 src2));
12852
12853 ins_cost(INSN_COST * 5);
12854 format %{ "fsubd $dst, $src1, $src2" %}
12855
12856 ins_encode %{
12857 __ fsubd(as_FloatRegister($dst$$reg),
12858 as_FloatRegister($src1$$reg),
12859 as_FloatRegister($src2$$reg));
12860 %}
12861
12862 ins_pipe(fp_dop_reg_reg_d);
12863 %}
12864
12865 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12866 match(Set dst (MulHF src1 src2));
12867 format %{ "fmulh $dst, $src1, $src2" %}
12868 ins_encode %{
12869 __ fmulh($dst$$FloatRegister,
12870 $src1$$FloatRegister,
12871 $src2$$FloatRegister);
12872 %}
12873 ins_pipe(fp_dop_reg_reg_s);
12874 %}
12875
12876 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12877 match(Set dst (MulF src1 src2));
12878
12879 ins_cost(INSN_COST * 6);
12880 format %{ "fmuls $dst, $src1, $src2" %}
12881
12882 ins_encode %{
12883 __ fmuls(as_FloatRegister($dst$$reg),
12884 as_FloatRegister($src1$$reg),
12885 as_FloatRegister($src2$$reg));
12886 %}
12887
12888 ins_pipe(fp_dop_reg_reg_s);
12889 %}
12890
12891 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12892 match(Set dst (MulD src1 src2));
12893
12894 ins_cost(INSN_COST * 6);
12895 format %{ "fmuld $dst, $src1, $src2" %}
12896
12897 ins_encode %{
12898 __ fmuld(as_FloatRegister($dst$$reg),
12899 as_FloatRegister($src1$$reg),
12900 as_FloatRegister($src2$$reg));
12901 %}
12902
12903 ins_pipe(fp_dop_reg_reg_d);
12904 %}
12905
12906 // src1 * src2 + src3 (half-precision float)
12907 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12908 match(Set dst (FmaHF src3 (Binary src1 src2)));
12909 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12910 ins_encode %{
12911 assert(UseFMA, "Needs FMA instructions support.");
12912 __ fmaddh($dst$$FloatRegister,
12913 $src1$$FloatRegister,
12914 $src2$$FloatRegister,
12915 $src3$$FloatRegister);
12916 %}
12917 ins_pipe(pipe_class_default);
12918 %}
12919
12920 // src1 * src2 + src3
12921 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12922 match(Set dst (FmaF src3 (Binary src1 src2)));
12923
12924 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12925
12926 ins_encode %{
12927 assert(UseFMA, "Needs FMA instructions support.");
12928 __ fmadds(as_FloatRegister($dst$$reg),
12929 as_FloatRegister($src1$$reg),
12930 as_FloatRegister($src2$$reg),
12931 as_FloatRegister($src3$$reg));
12932 %}
12933
12934 ins_pipe(pipe_class_default);
12935 %}
12936
12937 // src1 * src2 + src3
12938 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12939 match(Set dst (FmaD src3 (Binary src1 src2)));
12940
12941 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12942
12943 ins_encode %{
12944 assert(UseFMA, "Needs FMA instructions support.");
12945 __ fmaddd(as_FloatRegister($dst$$reg),
12946 as_FloatRegister($src1$$reg),
12947 as_FloatRegister($src2$$reg),
12948 as_FloatRegister($src3$$reg));
12949 %}
12950
12951 ins_pipe(pipe_class_default);
12952 %}
12953
12954 // src1 * (-src2) + src3
12955 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12956 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12957 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12958
12959 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12960
12961 ins_encode %{
12962 assert(UseFMA, "Needs FMA instructions support.");
12963 __ fmsubs(as_FloatRegister($dst$$reg),
12964 as_FloatRegister($src1$$reg),
12965 as_FloatRegister($src2$$reg),
12966 as_FloatRegister($src3$$reg));
12967 %}
12968
12969 ins_pipe(pipe_class_default);
12970 %}
12971
12972 // src1 * (-src2) + src3
12973 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12974 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12975 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12976
12977 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12978
12979 ins_encode %{
12980 assert(UseFMA, "Needs FMA instructions support.");
12981 __ fmsubd(as_FloatRegister($dst$$reg),
12982 as_FloatRegister($src1$$reg),
12983 as_FloatRegister($src2$$reg),
12984 as_FloatRegister($src3$$reg));
12985 %}
12986
12987 ins_pipe(pipe_class_default);
12988 %}
12989
12990 // src1 * (-src2) - src3
12991 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12992 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12993 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12994
12995 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
12996
12997 ins_encode %{
12998 assert(UseFMA, "Needs FMA instructions support.");
12999 __ fnmadds(as_FloatRegister($dst$$reg),
13000 as_FloatRegister($src1$$reg),
13001 as_FloatRegister($src2$$reg),
13002 as_FloatRegister($src3$$reg));
13003 %}
13004
13005 ins_pipe(pipe_class_default);
13006 %}
13007
13008 // src1 * (-src2) - src3
13009 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13010 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13011 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13012
13013 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13014
13015 ins_encode %{
13016 assert(UseFMA, "Needs FMA instructions support.");
13017 __ fnmaddd(as_FloatRegister($dst$$reg),
13018 as_FloatRegister($src1$$reg),
13019 as_FloatRegister($src2$$reg),
13020 as_FloatRegister($src3$$reg));
13021 %}
13022
13023 ins_pipe(pipe_class_default);
13024 %}
13025
13026 // src1 * src2 - src3
13027 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13028 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13029
13030 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13031
13032 ins_encode %{
13033 assert(UseFMA, "Needs FMA instructions support.");
13034 __ fnmsubs(as_FloatRegister($dst$$reg),
13035 as_FloatRegister($src1$$reg),
13036 as_FloatRegister($src2$$reg),
13037 as_FloatRegister($src3$$reg));
13038 %}
13039
13040 ins_pipe(pipe_class_default);
13041 %}
13042
13043 // src1 * src2 - src3
13044 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13045 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13046
13047 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13048
13049 ins_encode %{
13050 assert(UseFMA, "Needs FMA instructions support.");
13051 // n.b. insn name should be fnmsubd
13052 __ fnmsub(as_FloatRegister($dst$$reg),
13053 as_FloatRegister($src1$$reg),
13054 as_FloatRegister($src2$$reg),
13055 as_FloatRegister($src3$$reg));
13056 %}
13057
13058 ins_pipe(pipe_class_default);
13059 %}
13060
13061 // Math.max(HH)H (half-precision float)
13062 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13063 match(Set dst (MaxHF src1 src2));
13064 format %{ "fmaxh $dst, $src1, $src2" %}
13065 ins_encode %{
13066 __ fmaxh($dst$$FloatRegister,
13067 $src1$$FloatRegister,
13068 $src2$$FloatRegister);
13069 %}
13070 ins_pipe(fp_dop_reg_reg_s);
13071 %}
13072
13073 // Math.min(HH)H (half-precision float)
13074 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13075 match(Set dst (MinHF src1 src2));
13076 format %{ "fminh $dst, $src1, $src2" %}
13077 ins_encode %{
13078 __ fminh($dst$$FloatRegister,
13079 $src1$$FloatRegister,
13080 $src2$$FloatRegister);
13081 %}
13082 ins_pipe(fp_dop_reg_reg_s);
13083 %}
13084
13085 // Math.max(FF)F
13086 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13087 match(Set dst (MaxF src1 src2));
13088
13089 format %{ "fmaxs $dst, $src1, $src2" %}
13090 ins_encode %{
13091 __ fmaxs(as_FloatRegister($dst$$reg),
13092 as_FloatRegister($src1$$reg),
13093 as_FloatRegister($src2$$reg));
13094 %}
13095
13096 ins_pipe(fp_dop_reg_reg_s);
13097 %}
13098
13099 // Math.min(FF)F
13100 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13101 match(Set dst (MinF src1 src2));
13102
13103 format %{ "fmins $dst, $src1, $src2" %}
13104 ins_encode %{
13105 __ fmins(as_FloatRegister($dst$$reg),
13106 as_FloatRegister($src1$$reg),
13107 as_FloatRegister($src2$$reg));
13108 %}
13109
13110 ins_pipe(fp_dop_reg_reg_s);
13111 %}
13112
13113 // Math.max(DD)D
13114 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13115 match(Set dst (MaxD src1 src2));
13116
13117 format %{ "fmaxd $dst, $src1, $src2" %}
13118 ins_encode %{
13119 __ fmaxd(as_FloatRegister($dst$$reg),
13120 as_FloatRegister($src1$$reg),
13121 as_FloatRegister($src2$$reg));
13122 %}
13123
13124 ins_pipe(fp_dop_reg_reg_d);
13125 %}
13126
13127 // Math.min(DD)D
13128 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13129 match(Set dst (MinD src1 src2));
13130
13131 format %{ "fmind $dst, $src1, $src2" %}
13132 ins_encode %{
13133 __ fmind(as_FloatRegister($dst$$reg),
13134 as_FloatRegister($src1$$reg),
13135 as_FloatRegister($src2$$reg));
13136 %}
13137
13138 ins_pipe(fp_dop_reg_reg_d);
13139 %}
13140
13141 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13142 match(Set dst (DivHF src1 src2));
13143 format %{ "fdivh $dst, $src1, $src2" %}
13144 ins_encode %{
13145 __ fdivh($dst$$FloatRegister,
13146 $src1$$FloatRegister,
13147 $src2$$FloatRegister);
13148 %}
13149 ins_pipe(fp_div_s);
13150 %}
13151
13152 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13153 match(Set dst (DivF src1 src2));
13154
13155 ins_cost(INSN_COST * 18);
13156 format %{ "fdivs $dst, $src1, $src2" %}
13157
13158 ins_encode %{
13159 __ fdivs(as_FloatRegister($dst$$reg),
13160 as_FloatRegister($src1$$reg),
13161 as_FloatRegister($src2$$reg));
13162 %}
13163
13164 ins_pipe(fp_div_s);
13165 %}
13166
13167 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13168 match(Set dst (DivD src1 src2));
13169
13170 ins_cost(INSN_COST * 32);
13171 format %{ "fdivd $dst, $src1, $src2" %}
13172
13173 ins_encode %{
13174 __ fdivd(as_FloatRegister($dst$$reg),
13175 as_FloatRegister($src1$$reg),
13176 as_FloatRegister($src2$$reg));
13177 %}
13178
13179 ins_pipe(fp_div_d);
13180 %}
13181
13182 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13183 match(Set dst (NegF src));
13184
13185 ins_cost(INSN_COST * 3);
13186 format %{ "fneg $dst, $src" %}
13187
13188 ins_encode %{
13189 __ fnegs(as_FloatRegister($dst$$reg),
13190 as_FloatRegister($src$$reg));
13191 %}
13192
13193 ins_pipe(fp_uop_s);
13194 %}
13195
13196 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13197 match(Set dst (NegD src));
13198
13199 ins_cost(INSN_COST * 3);
13200 format %{ "fnegd $dst, $src" %}
13201
13202 ins_encode %{
13203 __ fnegd(as_FloatRegister($dst$$reg),
13204 as_FloatRegister($src$$reg));
13205 %}
13206
13207 ins_pipe(fp_uop_d);
13208 %}
13209
13210 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13211 %{
13212 match(Set dst (AbsI src));
13213
13214 effect(KILL cr);
13215 ins_cost(INSN_COST * 2);
13216 format %{ "cmpw $src, zr\n\t"
13217 "cnegw $dst, $src, Assembler::LT\t# int abs"
13218 %}
13219
13220 ins_encode %{
13221 __ cmpw(as_Register($src$$reg), zr);
13222 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13223 %}
13224 ins_pipe(pipe_class_default);
13225 %}
13226
13227 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13228 %{
13229 match(Set dst (AbsL src));
13230
13231 effect(KILL cr);
13232 ins_cost(INSN_COST * 2);
13233 format %{ "cmp $src, zr\n\t"
13234 "cneg $dst, $src, Assembler::LT\t# long abs"
13235 %}
13236
13237 ins_encode %{
13238 __ cmp(as_Register($src$$reg), zr);
13239 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13240 %}
13241 ins_pipe(pipe_class_default);
13242 %}
13243
13244 instruct absF_reg(vRegF dst, vRegF src) %{
13245 match(Set dst (AbsF src));
13246
13247 ins_cost(INSN_COST * 3);
13248 format %{ "fabss $dst, $src" %}
13249 ins_encode %{
13250 __ fabss(as_FloatRegister($dst$$reg),
13251 as_FloatRegister($src$$reg));
13252 %}
13253
13254 ins_pipe(fp_uop_s);
13255 %}
13256
13257 instruct absD_reg(vRegD dst, vRegD src) %{
13258 match(Set dst (AbsD src));
13259
13260 ins_cost(INSN_COST * 3);
13261 format %{ "fabsd $dst, $src" %}
13262 ins_encode %{
13263 __ fabsd(as_FloatRegister($dst$$reg),
13264 as_FloatRegister($src$$reg));
13265 %}
13266
13267 ins_pipe(fp_uop_d);
13268 %}
13269
13270 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13271 match(Set dst (AbsF (SubF src1 src2)));
13272
13273 ins_cost(INSN_COST * 3);
13274 format %{ "fabds $dst, $src1, $src2" %}
13275 ins_encode %{
13276 __ fabds(as_FloatRegister($dst$$reg),
13277 as_FloatRegister($src1$$reg),
13278 as_FloatRegister($src2$$reg));
13279 %}
13280
13281 ins_pipe(fp_uop_s);
13282 %}
13283
13284 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13285 match(Set dst (AbsD (SubD src1 src2)));
13286
13287 ins_cost(INSN_COST * 3);
13288 format %{ "fabdd $dst, $src1, $src2" %}
13289 ins_encode %{
13290 __ fabdd(as_FloatRegister($dst$$reg),
13291 as_FloatRegister($src1$$reg),
13292 as_FloatRegister($src2$$reg));
13293 %}
13294
13295 ins_pipe(fp_uop_d);
13296 %}
13297
13298 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13299 match(Set dst (SqrtD src));
13300
13301 ins_cost(INSN_COST * 50);
13302 format %{ "fsqrtd $dst, $src" %}
13303 ins_encode %{
13304 __ fsqrtd(as_FloatRegister($dst$$reg),
13305 as_FloatRegister($src$$reg));
13306 %}
13307
13308 ins_pipe(fp_div_s);
13309 %}
13310
13311 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13312 match(Set dst (SqrtF src));
13313
13314 ins_cost(INSN_COST * 50);
13315 format %{ "fsqrts $dst, $src" %}
13316 ins_encode %{
13317 __ fsqrts(as_FloatRegister($dst$$reg),
13318 as_FloatRegister($src$$reg));
13319 %}
13320
13321 ins_pipe(fp_div_d);
13322 %}
13323
13324 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13325 match(Set dst (SqrtHF src));
13326 format %{ "fsqrth $dst, $src" %}
13327 ins_encode %{
13328 __ fsqrth($dst$$FloatRegister,
13329 $src$$FloatRegister);
13330 %}
13331 ins_pipe(fp_div_s);
13332 %}
13333
13334 // Math.rint, floor, ceil
13335 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13336 match(Set dst (RoundDoubleMode src rmode));
13337 format %{ "frint $dst, $src, $rmode" %}
13338 ins_encode %{
13339 switch ($rmode$$constant) {
13340 case RoundDoubleModeNode::rmode_rint:
13341 __ frintnd(as_FloatRegister($dst$$reg),
13342 as_FloatRegister($src$$reg));
13343 break;
13344 case RoundDoubleModeNode::rmode_floor:
13345 __ frintmd(as_FloatRegister($dst$$reg),
13346 as_FloatRegister($src$$reg));
13347 break;
13348 case RoundDoubleModeNode::rmode_ceil:
13349 __ frintpd(as_FloatRegister($dst$$reg),
13350 as_FloatRegister($src$$reg));
13351 break;
13352 }
13353 %}
13354 ins_pipe(fp_uop_d);
13355 %}
13356
13357 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13358 match(Set dst (CopySignD src1 (Binary src2 zero)));
13359 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13360 format %{ "CopySignD $dst $src1 $src2" %}
13361 ins_encode %{
13362 FloatRegister dst = as_FloatRegister($dst$$reg),
13363 src1 = as_FloatRegister($src1$$reg),
13364 src2 = as_FloatRegister($src2$$reg),
13365 zero = as_FloatRegister($zero$$reg);
13366 __ fnegd(dst, zero);
13367 __ bsl(dst, __ T8B, src2, src1);
13368 %}
13369 ins_pipe(fp_uop_d);
13370 %}
13371
13372 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13373 match(Set dst (CopySignF src1 src2));
13374 effect(TEMP_DEF dst, USE src1, USE src2);
13375 format %{ "CopySignF $dst $src1 $src2" %}
13376 ins_encode %{
13377 FloatRegister dst = as_FloatRegister($dst$$reg),
13378 src1 = as_FloatRegister($src1$$reg),
13379 src2 = as_FloatRegister($src2$$reg);
13380 __ movi(dst, __ T2S, 0x80, 24);
13381 __ bsl(dst, __ T8B, src2, src1);
13382 %}
13383 ins_pipe(fp_uop_d);
13384 %}
13385
13386 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13387 match(Set dst (SignumD src (Binary zero one)));
13388 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13389 format %{ "signumD $dst, $src" %}
13390 ins_encode %{
13391 FloatRegister src = as_FloatRegister($src$$reg),
13392 dst = as_FloatRegister($dst$$reg),
13393 zero = as_FloatRegister($zero$$reg),
13394 one = as_FloatRegister($one$$reg);
13395 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13396 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13397 // Bit selection instruction gets bit from "one" for each enabled bit in
13398 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13399 // NaN the whole "src" will be copied because "dst" is zero. For all other
13400 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13401 // from "src", and all other bits are copied from 1.0.
13402 __ bsl(dst, __ T8B, one, src);
13403 %}
13404 ins_pipe(fp_uop_d);
13405 %}
13406
13407 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13408 match(Set dst (SignumF src (Binary zero one)));
13409 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13410 format %{ "signumF $dst, $src" %}
13411 ins_encode %{
13412 FloatRegister src = as_FloatRegister($src$$reg),
13413 dst = as_FloatRegister($dst$$reg),
13414 zero = as_FloatRegister($zero$$reg),
13415 one = as_FloatRegister($one$$reg);
13416 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13417 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13418 // Bit selection instruction gets bit from "one" for each enabled bit in
13419 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13420 // NaN the whole "src" will be copied because "dst" is zero. For all other
13421 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13422 // from "src", and all other bits are copied from 1.0.
13423 __ bsl(dst, __ T8B, one, src);
13424 %}
13425 ins_pipe(fp_uop_d);
13426 %}
13427
13428 instruct onspinwait() %{
13429 match(OnSpinWait);
13430 ins_cost(INSN_COST);
13431
13432 format %{ "onspinwait" %}
13433
13434 ins_encode %{
13435 __ spin_wait();
13436 %}
13437 ins_pipe(pipe_class_empty);
13438 %}
13439
13440 // ============================================================================
13441 // Logical Instructions
13442
13443 // Integer Logical Instructions
13444
13445 // And Instructions
13446
13447
13448 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13449 match(Set dst (AndI src1 src2));
13450
13451 format %{ "andw $dst, $src1, $src2\t# int" %}
13452
13453 ins_cost(INSN_COST);
13454 ins_encode %{
13455 __ andw(as_Register($dst$$reg),
13456 as_Register($src1$$reg),
13457 as_Register($src2$$reg));
13458 %}
13459
13460 ins_pipe(ialu_reg_reg);
13461 %}
13462
13463 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13464 match(Set dst (AndI src1 src2));
13465
13466 format %{ "andsw $dst, $src1, $src2\t# int" %}
13467
13468 ins_cost(INSN_COST);
13469 ins_encode %{
13470 __ andw(as_Register($dst$$reg),
13471 as_Register($src1$$reg),
13472 (uint64_t)($src2$$constant));
13473 %}
13474
13475 ins_pipe(ialu_reg_imm);
13476 %}
13477
13478 // Or Instructions
13479
13480 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13481 match(Set dst (OrI src1 src2));
13482
13483 format %{ "orrw $dst, $src1, $src2\t# int" %}
13484
13485 ins_cost(INSN_COST);
13486 ins_encode %{
13487 __ orrw(as_Register($dst$$reg),
13488 as_Register($src1$$reg),
13489 as_Register($src2$$reg));
13490 %}
13491
13492 ins_pipe(ialu_reg_reg);
13493 %}
13494
13495 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13496 match(Set dst (OrI src1 src2));
13497
13498 format %{ "orrw $dst, $src1, $src2\t# int" %}
13499
13500 ins_cost(INSN_COST);
13501 ins_encode %{
13502 __ orrw(as_Register($dst$$reg),
13503 as_Register($src1$$reg),
13504 (uint64_t)($src2$$constant));
13505 %}
13506
13507 ins_pipe(ialu_reg_imm);
13508 %}
13509
13510 // Xor Instructions
13511
13512 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13513 match(Set dst (XorI src1 src2));
13514
13515 format %{ "eorw $dst, $src1, $src2\t# int" %}
13516
13517 ins_cost(INSN_COST);
13518 ins_encode %{
13519 __ eorw(as_Register($dst$$reg),
13520 as_Register($src1$$reg),
13521 as_Register($src2$$reg));
13522 %}
13523
13524 ins_pipe(ialu_reg_reg);
13525 %}
13526
13527 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13528 match(Set dst (XorI src1 src2));
13529
13530 format %{ "eorw $dst, $src1, $src2\t# int" %}
13531
13532 ins_cost(INSN_COST);
13533 ins_encode %{
13534 __ eorw(as_Register($dst$$reg),
13535 as_Register($src1$$reg),
13536 (uint64_t)($src2$$constant));
13537 %}
13538
13539 ins_pipe(ialu_reg_imm);
13540 %}
13541
13542 // Long Logical Instructions
13543 // TODO
13544
13545 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13546 match(Set dst (AndL src1 src2));
13547
13548 format %{ "and $dst, $src1, $src2\t# int" %}
13549
13550 ins_cost(INSN_COST);
13551 ins_encode %{
13552 __ andr(as_Register($dst$$reg),
13553 as_Register($src1$$reg),
13554 as_Register($src2$$reg));
13555 %}
13556
13557 ins_pipe(ialu_reg_reg);
13558 %}
13559
13560 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13561 match(Set dst (AndL src1 src2));
13562
13563 format %{ "and $dst, $src1, $src2\t# int" %}
13564
13565 ins_cost(INSN_COST);
13566 ins_encode %{
13567 __ andr(as_Register($dst$$reg),
13568 as_Register($src1$$reg),
13569 (uint64_t)($src2$$constant));
13570 %}
13571
13572 ins_pipe(ialu_reg_imm);
13573 %}
13574
13575 // Or Instructions
13576
13577 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13578 match(Set dst (OrL src1 src2));
13579
13580 format %{ "orr $dst, $src1, $src2\t# int" %}
13581
13582 ins_cost(INSN_COST);
13583 ins_encode %{
13584 __ orr(as_Register($dst$$reg),
13585 as_Register($src1$$reg),
13586 as_Register($src2$$reg));
13587 %}
13588
13589 ins_pipe(ialu_reg_reg);
13590 %}
13591
13592 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13593 match(Set dst (OrL src1 src2));
13594
13595 format %{ "orr $dst, $src1, $src2\t# int" %}
13596
13597 ins_cost(INSN_COST);
13598 ins_encode %{
13599 __ orr(as_Register($dst$$reg),
13600 as_Register($src1$$reg),
13601 (uint64_t)($src2$$constant));
13602 %}
13603
13604 ins_pipe(ialu_reg_imm);
13605 %}
13606
13607 // Xor Instructions
13608
13609 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13610 match(Set dst (XorL src1 src2));
13611
13612 format %{ "eor $dst, $src1, $src2\t# int" %}
13613
13614 ins_cost(INSN_COST);
13615 ins_encode %{
13616 __ eor(as_Register($dst$$reg),
13617 as_Register($src1$$reg),
13618 as_Register($src2$$reg));
13619 %}
13620
13621 ins_pipe(ialu_reg_reg);
13622 %}
13623
13624 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13625 match(Set dst (XorL src1 src2));
13626
13627 ins_cost(INSN_COST);
13628 format %{ "eor $dst, $src1, $src2\t# int" %}
13629
13630 ins_encode %{
13631 __ eor(as_Register($dst$$reg),
13632 as_Register($src1$$reg),
13633 (uint64_t)($src2$$constant));
13634 %}
13635
13636 ins_pipe(ialu_reg_imm);
13637 %}
13638
13639 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13640 %{
13641 match(Set dst (ConvI2L src));
13642
13643 ins_cost(INSN_COST);
13644 format %{ "sxtw $dst, $src\t# i2l" %}
13645 ins_encode %{
13646 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13647 %}
13648 ins_pipe(ialu_reg_shift);
13649 %}
13650
13651 // this pattern occurs in bigmath arithmetic
13652 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13653 %{
13654 match(Set dst (AndL (ConvI2L src) mask));
13655
13656 ins_cost(INSN_COST);
13657 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13658 ins_encode %{
13659 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13660 %}
13661
13662 ins_pipe(ialu_reg_shift);
13663 %}
13664
13665 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13666 match(Set dst (ConvL2I src));
13667
13668 ins_cost(INSN_COST);
13669 format %{ "movw $dst, $src \t// l2i" %}
13670
13671 ins_encode %{
13672 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13673 %}
13674
13675 ins_pipe(ialu_reg);
13676 %}
13677
13678 instruct convD2F_reg(vRegF dst, vRegD src) %{
13679 match(Set dst (ConvD2F src));
13680
13681 ins_cost(INSN_COST * 5);
13682 format %{ "fcvtd $dst, $src \t// d2f" %}
13683
13684 ins_encode %{
13685 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13686 %}
13687
13688 ins_pipe(fp_d2f);
13689 %}
13690
13691 instruct convF2D_reg(vRegD dst, vRegF src) %{
13692 match(Set dst (ConvF2D src));
13693
13694 ins_cost(INSN_COST * 5);
13695 format %{ "fcvts $dst, $src \t// f2d" %}
13696
13697 ins_encode %{
13698 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13699 %}
13700
13701 ins_pipe(fp_f2d);
13702 %}
13703
13704 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13705 match(Set dst (ConvF2I src));
13706
13707 ins_cost(INSN_COST * 5);
13708 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13709
13710 ins_encode %{
13711 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13712 %}
13713
13714 ins_pipe(fp_f2i);
13715 %}
13716
13717 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13718 match(Set dst (ConvF2L src));
13719
13720 ins_cost(INSN_COST * 5);
13721 format %{ "fcvtzs $dst, $src \t// f2l" %}
13722
13723 ins_encode %{
13724 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13725 %}
13726
13727 ins_pipe(fp_f2l);
13728 %}
13729
13730 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13731 match(Set dst (ConvF2HF src));
13732 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13733 "smov $dst, $tmp\t# move result from $tmp to $dst"
13734 %}
13735 effect(TEMP tmp);
13736 ins_encode %{
13737 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13738 %}
13739 ins_pipe(pipe_slow);
13740 %}
13741
13742 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13743 match(Set dst (ConvHF2F src));
13744 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13745 "fcvt $dst, $tmp\t# convert half to single precision"
13746 %}
13747 effect(TEMP tmp);
13748 ins_encode %{
13749 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13750 %}
13751 ins_pipe(pipe_slow);
13752 %}
13753
13754 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13755 match(Set dst (ConvI2F src));
13756
13757 ins_cost(INSN_COST * 5);
13758 format %{ "scvtfws $dst, $src \t// i2f" %}
13759
13760 ins_encode %{
13761 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13762 %}
13763
13764 ins_pipe(fp_i2f);
13765 %}
13766
13767 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13768 match(Set dst (ConvL2F src));
13769
13770 ins_cost(INSN_COST * 5);
13771 format %{ "scvtfs $dst, $src \t// l2f" %}
13772
13773 ins_encode %{
13774 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13775 %}
13776
13777 ins_pipe(fp_l2f);
13778 %}
13779
13780 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13781 match(Set dst (ConvD2I src));
13782
13783 ins_cost(INSN_COST * 5);
13784 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13785
13786 ins_encode %{
13787 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13788 %}
13789
13790 ins_pipe(fp_d2i);
13791 %}
13792
13793 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13794 match(Set dst (ConvD2L src));
13795
13796 ins_cost(INSN_COST * 5);
13797 format %{ "fcvtzd $dst, $src \t// d2l" %}
13798
13799 ins_encode %{
13800 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13801 %}
13802
13803 ins_pipe(fp_d2l);
13804 %}
13805
13806 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13807 match(Set dst (ConvI2D src));
13808
13809 ins_cost(INSN_COST * 5);
13810 format %{ "scvtfwd $dst, $src \t// i2d" %}
13811
13812 ins_encode %{
13813 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13814 %}
13815
13816 ins_pipe(fp_i2d);
13817 %}
13818
13819 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13820 match(Set dst (ConvL2D src));
13821
13822 ins_cost(INSN_COST * 5);
13823 format %{ "scvtfd $dst, $src \t// l2d" %}
13824
13825 ins_encode %{
13826 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13827 %}
13828
13829 ins_pipe(fp_l2d);
13830 %}
13831
13832 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13833 %{
13834 match(Set dst (RoundD src));
13835 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13836 format %{ "java_round_double $dst,$src"%}
13837 ins_encode %{
13838 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13839 as_FloatRegister($ftmp$$reg));
13840 %}
13841 ins_pipe(pipe_slow);
13842 %}
13843
13844 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13845 %{
13846 match(Set dst (RoundF src));
13847 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13848 format %{ "java_round_float $dst,$src"%}
13849 ins_encode %{
13850 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13851 as_FloatRegister($ftmp$$reg));
13852 %}
13853 ins_pipe(pipe_slow);
13854 %}
13855
13856 // stack <-> reg and reg <-> reg shuffles with no conversion
13857
13858 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13859
13860 match(Set dst (MoveF2I src));
13861
13862 effect(DEF dst, USE src);
13863
13864 ins_cost(4 * INSN_COST);
13865
13866 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13867
13868 ins_encode %{
13869 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13870 %}
13871
13872 ins_pipe(iload_reg_reg);
13873
13874 %}
13875
13876 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13877
13878 match(Set dst (MoveI2F src));
13879
13880 effect(DEF dst, USE src);
13881
13882 ins_cost(4 * INSN_COST);
13883
13884 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13885
13886 ins_encode %{
13887 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13888 %}
13889
13890 ins_pipe(pipe_class_memory);
13891
13892 %}
13893
13894 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13895
13896 match(Set dst (MoveD2L src));
13897
13898 effect(DEF dst, USE src);
13899
13900 ins_cost(4 * INSN_COST);
13901
13902 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13903
13904 ins_encode %{
13905 __ ldr($dst$$Register, Address(sp, $src$$disp));
13906 %}
13907
13908 ins_pipe(iload_reg_reg);
13909
13910 %}
13911
13912 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13913
13914 match(Set dst (MoveL2D src));
13915
13916 effect(DEF dst, USE src);
13917
13918 ins_cost(4 * INSN_COST);
13919
13920 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13921
13922 ins_encode %{
13923 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13924 %}
13925
13926 ins_pipe(pipe_class_memory);
13927
13928 %}
13929
13930 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13931
13932 match(Set dst (MoveF2I src));
13933
13934 effect(DEF dst, USE src);
13935
13936 ins_cost(INSN_COST);
13937
13938 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13939
13940 ins_encode %{
13941 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13942 %}
13943
13944 ins_pipe(pipe_class_memory);
13945
13946 %}
13947
13948 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13949
13950 match(Set dst (MoveI2F src));
13951
13952 effect(DEF dst, USE src);
13953
13954 ins_cost(INSN_COST);
13955
13956 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13957
13958 ins_encode %{
13959 __ strw($src$$Register, Address(sp, $dst$$disp));
13960 %}
13961
13962 ins_pipe(istore_reg_reg);
13963
13964 %}
13965
13966 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13967
13968 match(Set dst (MoveD2L src));
13969
13970 effect(DEF dst, USE src);
13971
13972 ins_cost(INSN_COST);
13973
13974 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13975
13976 ins_encode %{
13977 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13978 %}
13979
13980 ins_pipe(pipe_class_memory);
13981
13982 %}
13983
13984 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13985
13986 match(Set dst (MoveL2D src));
13987
13988 effect(DEF dst, USE src);
13989
13990 ins_cost(INSN_COST);
13991
13992 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13993
13994 ins_encode %{
13995 __ str($src$$Register, Address(sp, $dst$$disp));
13996 %}
13997
13998 ins_pipe(istore_reg_reg);
13999
14000 %}
14001
14002 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14003
14004 match(Set dst (MoveF2I src));
14005
14006 effect(DEF dst, USE src);
14007
14008 ins_cost(INSN_COST);
14009
14010 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14011
14012 ins_encode %{
14013 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14014 %}
14015
14016 ins_pipe(fp_f2i);
14017
14018 %}
14019
14020 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14021
14022 match(Set dst (MoveI2F src));
14023
14024 effect(DEF dst, USE src);
14025
14026 ins_cost(INSN_COST);
14027
14028 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14029
14030 ins_encode %{
14031 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14032 %}
14033
14034 ins_pipe(fp_i2f);
14035
14036 %}
14037
14038 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14039
14040 match(Set dst (MoveD2L src));
14041
14042 effect(DEF dst, USE src);
14043
14044 ins_cost(INSN_COST);
14045
14046 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14047
14048 ins_encode %{
14049 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14050 %}
14051
14052 ins_pipe(fp_d2l);
14053
14054 %}
14055
14056 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14057
14058 match(Set dst (MoveL2D src));
14059
14060 effect(DEF dst, USE src);
14061
14062 ins_cost(INSN_COST);
14063
14064 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14065
14066 ins_encode %{
14067 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14068 %}
14069
14070 ins_pipe(fp_l2d);
14071
14072 %}
14073
14074 // ============================================================================
14075 // clearing of an array
14076
14077 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14078 %{
14079 match(Set dummy (ClearArray cnt base));
14080 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14081
14082 ins_cost(4 * INSN_COST);
14083 format %{ "ClearArray $cnt, $base" %}
14084
14085 ins_encode %{
14086 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14087 if (tpc == nullptr) {
14088 ciEnv::current()->record_failure("CodeCache is full");
14089 return;
14090 }
14091 %}
14092
14093 ins_pipe(pipe_class_memory);
14094 %}
14095
14096 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14097 %{
14098 predicate((uint64_t)n->in(2)->get_long()
14099 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14100 match(Set dummy (ClearArray cnt base));
14101 effect(TEMP temp, USE_KILL base, KILL cr);
14102
14103 ins_cost(4 * INSN_COST);
14104 format %{ "ClearArray $cnt, $base" %}
14105
14106 ins_encode %{
14107 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14108 if (tpc == nullptr) {
14109 ciEnv::current()->record_failure("CodeCache is full");
14110 return;
14111 }
14112 %}
14113
14114 ins_pipe(pipe_class_memory);
14115 %}
14116
14117 // ============================================================================
14118 // Overflow Math Instructions
14119
14120 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14121 %{
14122 match(Set cr (OverflowAddI op1 op2));
14123
14124 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14125 ins_cost(INSN_COST);
14126 ins_encode %{
14127 __ cmnw($op1$$Register, $op2$$Register);
14128 %}
14129
14130 ins_pipe(icmp_reg_reg);
14131 %}
14132
14133 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14134 %{
14135 match(Set cr (OverflowAddI op1 op2));
14136
14137 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14138 ins_cost(INSN_COST);
14139 ins_encode %{
14140 __ cmnw($op1$$Register, $op2$$constant);
14141 %}
14142
14143 ins_pipe(icmp_reg_imm);
14144 %}
14145
14146 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14147 %{
14148 match(Set cr (OverflowAddL op1 op2));
14149
14150 format %{ "cmn $op1, $op2\t# overflow check long" %}
14151 ins_cost(INSN_COST);
14152 ins_encode %{
14153 __ cmn($op1$$Register, $op2$$Register);
14154 %}
14155
14156 ins_pipe(icmp_reg_reg);
14157 %}
14158
14159 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14160 %{
14161 match(Set cr (OverflowAddL op1 op2));
14162
14163 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14164 ins_cost(INSN_COST);
14165 ins_encode %{
14166 __ adds(zr, $op1$$Register, $op2$$constant);
14167 %}
14168
14169 ins_pipe(icmp_reg_imm);
14170 %}
14171
14172 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14173 %{
14174 match(Set cr (OverflowSubI op1 op2));
14175
14176 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14177 ins_cost(INSN_COST);
14178 ins_encode %{
14179 __ cmpw($op1$$Register, $op2$$Register);
14180 %}
14181
14182 ins_pipe(icmp_reg_reg);
14183 %}
14184
14185 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14186 %{
14187 match(Set cr (OverflowSubI op1 op2));
14188
14189 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14190 ins_cost(INSN_COST);
14191 ins_encode %{
14192 __ cmpw($op1$$Register, $op2$$constant);
14193 %}
14194
14195 ins_pipe(icmp_reg_imm);
14196 %}
14197
14198 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14199 %{
14200 match(Set cr (OverflowSubL op1 op2));
14201
14202 format %{ "cmp $op1, $op2\t# overflow check long" %}
14203 ins_cost(INSN_COST);
14204 ins_encode %{
14205 __ cmp($op1$$Register, $op2$$Register);
14206 %}
14207
14208 ins_pipe(icmp_reg_reg);
14209 %}
14210
14211 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14212 %{
14213 match(Set cr (OverflowSubL op1 op2));
14214
14215 format %{ "cmp $op1, $op2\t# overflow check long" %}
14216 ins_cost(INSN_COST);
14217 ins_encode %{
14218 __ subs(zr, $op1$$Register, $op2$$constant);
14219 %}
14220
14221 ins_pipe(icmp_reg_imm);
14222 %}
14223
14224 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14225 %{
14226 match(Set cr (OverflowSubI zero op1));
14227
14228 format %{ "cmpw zr, $op1\t# overflow check int" %}
14229 ins_cost(INSN_COST);
14230 ins_encode %{
14231 __ cmpw(zr, $op1$$Register);
14232 %}
14233
14234 ins_pipe(icmp_reg_imm);
14235 %}
14236
14237 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14238 %{
14239 match(Set cr (OverflowSubL zero op1));
14240
14241 format %{ "cmp zr, $op1\t# overflow check long" %}
14242 ins_cost(INSN_COST);
14243 ins_encode %{
14244 __ cmp(zr, $op1$$Register);
14245 %}
14246
14247 ins_pipe(icmp_reg_imm);
14248 %}
14249
14250 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14251 %{
14252 match(Set cr (OverflowMulI op1 op2));
14253
14254 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14255 "cmp rscratch1, rscratch1, sxtw\n\t"
14256 "movw rscratch1, #0x80000000\n\t"
14257 "cselw rscratch1, rscratch1, zr, NE\n\t"
14258 "cmpw rscratch1, #1" %}
14259 ins_cost(5 * INSN_COST);
14260 ins_encode %{
14261 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14262 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14263 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14264 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14265 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14266 %}
14267
14268 ins_pipe(pipe_slow);
14269 %}
14270
14271 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14272 %{
14273 match(If cmp (OverflowMulI op1 op2));
14274 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14275 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14276 effect(USE labl, KILL cr);
14277
14278 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14279 "cmp rscratch1, rscratch1, sxtw\n\t"
14280 "b$cmp $labl" %}
14281 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14282 ins_encode %{
14283 Label* L = $labl$$label;
14284 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14285 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14286 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14287 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14288 %}
14289
14290 ins_pipe(pipe_serial);
14291 %}
14292
14293 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14294 %{
14295 match(Set cr (OverflowMulL op1 op2));
14296
14297 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14298 "smulh rscratch2, $op1, $op2\n\t"
14299 "cmp rscratch2, rscratch1, ASR #63\n\t"
14300 "movw rscratch1, #0x80000000\n\t"
14301 "cselw rscratch1, rscratch1, zr, NE\n\t"
14302 "cmpw rscratch1, #1" %}
14303 ins_cost(6 * INSN_COST);
14304 ins_encode %{
14305 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14306 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14307 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14308 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14309 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14310 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14311 %}
14312
14313 ins_pipe(pipe_slow);
14314 %}
14315
14316 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14317 %{
14318 match(If cmp (OverflowMulL op1 op2));
14319 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14320 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14321 effect(USE labl, KILL cr);
14322
14323 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14324 "smulh rscratch2, $op1, $op2\n\t"
14325 "cmp rscratch2, rscratch1, ASR #63\n\t"
14326 "b$cmp $labl" %}
14327 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14328 ins_encode %{
14329 Label* L = $labl$$label;
14330 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14331 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14332 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14333 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14334 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14335 %}
14336
14337 ins_pipe(pipe_serial);
14338 %}
14339
14340 // ============================================================================
14341 // Compare Instructions
14342
14343 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14344 %{
14345 match(Set cr (CmpI op1 op2));
14346
14347 effect(DEF cr, USE op1, USE op2);
14348
14349 ins_cost(INSN_COST);
14350 format %{ "cmpw $op1, $op2" %}
14351
14352 ins_encode(aarch64_enc_cmpw(op1, op2));
14353
14354 ins_pipe(icmp_reg_reg);
14355 %}
14356
14357 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14358 %{
14359 match(Set cr (CmpI op1 zero));
14360
14361 effect(DEF cr, USE op1);
14362
14363 ins_cost(INSN_COST);
14364 format %{ "cmpw $op1, 0" %}
14365
14366 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14367
14368 ins_pipe(icmp_reg_imm);
14369 %}
14370
14371 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14372 %{
14373 match(Set cr (CmpI op1 op2));
14374
14375 effect(DEF cr, USE op1);
14376
14377 ins_cost(INSN_COST);
14378 format %{ "cmpw $op1, $op2" %}
14379
14380 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14381
14382 ins_pipe(icmp_reg_imm);
14383 %}
14384
14385 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14386 %{
14387 match(Set cr (CmpI op1 op2));
14388
14389 effect(DEF cr, USE op1);
14390
14391 ins_cost(INSN_COST * 2);
14392 format %{ "cmpw $op1, $op2" %}
14393
14394 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14395
14396 ins_pipe(icmp_reg_imm);
14397 %}
14398
14399 // Unsigned compare Instructions; really, same as signed compare
14400 // except it should only be used to feed an If or a CMovI which takes a
14401 // cmpOpU.
14402
14403 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14404 %{
14405 match(Set cr (CmpU op1 op2));
14406
14407 effect(DEF cr, USE op1, USE op2);
14408
14409 ins_cost(INSN_COST);
14410 format %{ "cmpw $op1, $op2\t# unsigned" %}
14411
14412 ins_encode(aarch64_enc_cmpw(op1, op2));
14413
14414 ins_pipe(icmp_reg_reg);
14415 %}
14416
14417 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14418 %{
14419 match(Set cr (CmpU op1 zero));
14420
14421 effect(DEF cr, USE op1);
14422
14423 ins_cost(INSN_COST);
14424 format %{ "cmpw $op1, #0\t# unsigned" %}
14425
14426 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14427
14428 ins_pipe(icmp_reg_imm);
14429 %}
14430
14431 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14432 %{
14433 match(Set cr (CmpU op1 op2));
14434
14435 effect(DEF cr, USE op1);
14436
14437 ins_cost(INSN_COST);
14438 format %{ "cmpw $op1, $op2\t# unsigned" %}
14439
14440 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14441
14442 ins_pipe(icmp_reg_imm);
14443 %}
14444
14445 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14446 %{
14447 match(Set cr (CmpU op1 op2));
14448
14449 effect(DEF cr, USE op1);
14450
14451 ins_cost(INSN_COST * 2);
14452 format %{ "cmpw $op1, $op2\t# unsigned" %}
14453
14454 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14455
14456 ins_pipe(icmp_reg_imm);
14457 %}
14458
14459 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14460 %{
14461 match(Set cr (CmpL op1 op2));
14462
14463 effect(DEF cr, USE op1, USE op2);
14464
14465 ins_cost(INSN_COST);
14466 format %{ "cmp $op1, $op2" %}
14467
14468 ins_encode(aarch64_enc_cmp(op1, op2));
14469
14470 ins_pipe(icmp_reg_reg);
14471 %}
14472
14473 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14474 %{
14475 match(Set cr (CmpL op1 zero));
14476
14477 effect(DEF cr, USE op1);
14478
14479 ins_cost(INSN_COST);
14480 format %{ "tst $op1" %}
14481
14482 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14483
14484 ins_pipe(icmp_reg_imm);
14485 %}
14486
14487 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14488 %{
14489 match(Set cr (CmpL op1 op2));
14490
14491 effect(DEF cr, USE op1);
14492
14493 ins_cost(INSN_COST);
14494 format %{ "cmp $op1, $op2" %}
14495
14496 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14497
14498 ins_pipe(icmp_reg_imm);
14499 %}
14500
14501 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14502 %{
14503 match(Set cr (CmpL op1 op2));
14504
14505 effect(DEF cr, USE op1);
14506
14507 ins_cost(INSN_COST * 2);
14508 format %{ "cmp $op1, $op2" %}
14509
14510 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14511
14512 ins_pipe(icmp_reg_imm);
14513 %}
14514
14515 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14516 %{
14517 match(Set cr (CmpUL op1 op2));
14518
14519 effect(DEF cr, USE op1, USE op2);
14520
14521 ins_cost(INSN_COST);
14522 format %{ "cmp $op1, $op2" %}
14523
14524 ins_encode(aarch64_enc_cmp(op1, op2));
14525
14526 ins_pipe(icmp_reg_reg);
14527 %}
14528
14529 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14530 %{
14531 match(Set cr (CmpUL op1 zero));
14532
14533 effect(DEF cr, USE op1);
14534
14535 ins_cost(INSN_COST);
14536 format %{ "tst $op1" %}
14537
14538 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14539
14540 ins_pipe(icmp_reg_imm);
14541 %}
14542
14543 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14544 %{
14545 match(Set cr (CmpUL op1 op2));
14546
14547 effect(DEF cr, USE op1);
14548
14549 ins_cost(INSN_COST);
14550 format %{ "cmp $op1, $op2" %}
14551
14552 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14553
14554 ins_pipe(icmp_reg_imm);
14555 %}
14556
14557 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14558 %{
14559 match(Set cr (CmpUL op1 op2));
14560
14561 effect(DEF cr, USE op1);
14562
14563 ins_cost(INSN_COST * 2);
14564 format %{ "cmp $op1, $op2" %}
14565
14566 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14567
14568 ins_pipe(icmp_reg_imm);
14569 %}
14570
14571 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14572 %{
14573 match(Set cr (CmpP op1 op2));
14574
14575 effect(DEF cr, USE op1, USE op2);
14576
14577 ins_cost(INSN_COST);
14578 format %{ "cmp $op1, $op2\t // ptr" %}
14579
14580 ins_encode(aarch64_enc_cmpp(op1, op2));
14581
14582 ins_pipe(icmp_reg_reg);
14583 %}
14584
14585 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14586 %{
14587 match(Set cr (CmpN op1 op2));
14588
14589 effect(DEF cr, USE op1, USE op2);
14590
14591 ins_cost(INSN_COST);
14592 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14593
14594 ins_encode(aarch64_enc_cmpn(op1, op2));
14595
14596 ins_pipe(icmp_reg_reg);
14597 %}
14598
14599 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14600 %{
14601 match(Set cr (CmpP op1 zero));
14602
14603 effect(DEF cr, USE op1, USE zero);
14604
14605 ins_cost(INSN_COST);
14606 format %{ "cmp $op1, 0\t // ptr" %}
14607
14608 ins_encode(aarch64_enc_testp(op1));
14609
14610 ins_pipe(icmp_reg_imm);
14611 %}
14612
14613 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14614 %{
14615 match(Set cr (CmpN op1 zero));
14616
14617 effect(DEF cr, USE op1, USE zero);
14618
14619 ins_cost(INSN_COST);
14620 format %{ "cmp $op1, 0\t // compressed ptr" %}
14621
14622 ins_encode(aarch64_enc_testn(op1));
14623
14624 ins_pipe(icmp_reg_imm);
14625 %}
14626
14627 // FP comparisons
14628 //
14629 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14630 // using normal cmpOp. See declaration of rFlagsReg for details.
14631
14632 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14633 %{
14634 match(Set cr (CmpF src1 src2));
14635
14636 ins_cost(3 * INSN_COST);
14637 format %{ "fcmps $src1, $src2" %}
14638
14639 ins_encode %{
14640 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14641 %}
14642
14643 ins_pipe(pipe_class_compare);
14644 %}
14645
14646 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14647 %{
14648 match(Set cr (CmpF src1 src2));
14649
14650 ins_cost(3 * INSN_COST);
14651 format %{ "fcmps $src1, 0.0" %}
14652
14653 ins_encode %{
14654 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14655 %}
14656
14657 ins_pipe(pipe_class_compare);
14658 %}
14659 // FROM HERE
14660
14661 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14662 %{
14663 match(Set cr (CmpD src1 src2));
14664
14665 ins_cost(3 * INSN_COST);
14666 format %{ "fcmpd $src1, $src2" %}
14667
14668 ins_encode %{
14669 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14670 %}
14671
14672 ins_pipe(pipe_class_compare);
14673 %}
14674
14675 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14676 %{
14677 match(Set cr (CmpD src1 src2));
14678
14679 ins_cost(3 * INSN_COST);
14680 format %{ "fcmpd $src1, 0.0" %}
14681
14682 ins_encode %{
14683 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14684 %}
14685
14686 ins_pipe(pipe_class_compare);
14687 %}
14688
14689 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14690 %{
14691 match(Set dst (CmpF3 src1 src2));
14692 effect(KILL cr);
14693
14694 ins_cost(5 * INSN_COST);
14695 format %{ "fcmps $src1, $src2\n\t"
14696 "csinvw($dst, zr, zr, eq\n\t"
14697 "csnegw($dst, $dst, $dst, lt)"
14698 %}
14699
14700 ins_encode %{
14701 Label done;
14702 FloatRegister s1 = as_FloatRegister($src1$$reg);
14703 FloatRegister s2 = as_FloatRegister($src2$$reg);
14704 Register d = as_Register($dst$$reg);
14705 __ fcmps(s1, s2);
14706 // installs 0 if EQ else -1
14707 __ csinvw(d, zr, zr, Assembler::EQ);
14708 // keeps -1 if less or unordered else installs 1
14709 __ csnegw(d, d, d, Assembler::LT);
14710 __ bind(done);
14711 %}
14712
14713 ins_pipe(pipe_class_default);
14714
14715 %}
14716
14717 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14718 %{
14719 match(Set dst (CmpD3 src1 src2));
14720 effect(KILL cr);
14721
14722 ins_cost(5 * INSN_COST);
14723 format %{ "fcmpd $src1, $src2\n\t"
14724 "csinvw($dst, zr, zr, eq\n\t"
14725 "csnegw($dst, $dst, $dst, lt)"
14726 %}
14727
14728 ins_encode %{
14729 Label done;
14730 FloatRegister s1 = as_FloatRegister($src1$$reg);
14731 FloatRegister s2 = as_FloatRegister($src2$$reg);
14732 Register d = as_Register($dst$$reg);
14733 __ fcmpd(s1, s2);
14734 // installs 0 if EQ else -1
14735 __ csinvw(d, zr, zr, Assembler::EQ);
14736 // keeps -1 if less or unordered else installs 1
14737 __ csnegw(d, d, d, Assembler::LT);
14738 __ bind(done);
14739 %}
14740 ins_pipe(pipe_class_default);
14741
14742 %}
14743
14744 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14745 %{
14746 match(Set dst (CmpF3 src1 zero));
14747 effect(KILL cr);
14748
14749 ins_cost(5 * INSN_COST);
14750 format %{ "fcmps $src1, 0.0\n\t"
14751 "csinvw($dst, zr, zr, eq\n\t"
14752 "csnegw($dst, $dst, $dst, lt)"
14753 %}
14754
14755 ins_encode %{
14756 Label done;
14757 FloatRegister s1 = as_FloatRegister($src1$$reg);
14758 Register d = as_Register($dst$$reg);
14759 __ fcmps(s1, 0.0);
14760 // installs 0 if EQ else -1
14761 __ csinvw(d, zr, zr, Assembler::EQ);
14762 // keeps -1 if less or unordered else installs 1
14763 __ csnegw(d, d, d, Assembler::LT);
14764 __ bind(done);
14765 %}
14766
14767 ins_pipe(pipe_class_default);
14768
14769 %}
14770
14771 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14772 %{
14773 match(Set dst (CmpD3 src1 zero));
14774 effect(KILL cr);
14775
14776 ins_cost(5 * INSN_COST);
14777 format %{ "fcmpd $src1, 0.0\n\t"
14778 "csinvw($dst, zr, zr, eq\n\t"
14779 "csnegw($dst, $dst, $dst, lt)"
14780 %}
14781
14782 ins_encode %{
14783 Label done;
14784 FloatRegister s1 = as_FloatRegister($src1$$reg);
14785 Register d = as_Register($dst$$reg);
14786 __ fcmpd(s1, 0.0);
14787 // installs 0 if EQ else -1
14788 __ csinvw(d, zr, zr, Assembler::EQ);
14789 // keeps -1 if less or unordered else installs 1
14790 __ csnegw(d, d, d, Assembler::LT);
14791 __ bind(done);
14792 %}
14793 ins_pipe(pipe_class_default);
14794
14795 %}
14796
14797 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14798 %{
14799 match(Set dst (CmpLTMask p q));
14800 effect(KILL cr);
14801
14802 ins_cost(3 * INSN_COST);
14803
14804 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14805 "csetw $dst, lt\n\t"
14806 "subw $dst, zr, $dst"
14807 %}
14808
14809 ins_encode %{
14810 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14811 __ csetw(as_Register($dst$$reg), Assembler::LT);
14812 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14813 %}
14814
14815 ins_pipe(ialu_reg_reg);
14816 %}
14817
14818 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14819 %{
14820 match(Set dst (CmpLTMask src zero));
14821 effect(KILL cr);
14822
14823 ins_cost(INSN_COST);
14824
14825 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14826
14827 ins_encode %{
14828 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14829 %}
14830
14831 ins_pipe(ialu_reg_shift);
14832 %}
14833
14834 // ============================================================================
14835 // Max and Min
14836
14837 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14838
14839 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14840 %{
14841 effect(DEF cr, USE src);
14842 ins_cost(INSN_COST);
14843 format %{ "cmpw $src, 0" %}
14844
14845 ins_encode %{
14846 __ cmpw($src$$Register, 0);
14847 %}
14848 ins_pipe(icmp_reg_imm);
14849 %}
14850
14851 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14852 %{
14853 match(Set dst (MinI src1 src2));
14854 ins_cost(INSN_COST * 3);
14855
14856 expand %{
14857 rFlagsReg cr;
14858 compI_reg_reg(cr, src1, src2);
14859 cmovI_reg_reg_lt(dst, src1, src2, cr);
14860 %}
14861 %}
14862
14863 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14864 %{
14865 match(Set dst (MaxI src1 src2));
14866 ins_cost(INSN_COST * 3);
14867
14868 expand %{
14869 rFlagsReg cr;
14870 compI_reg_reg(cr, src1, src2);
14871 cmovI_reg_reg_gt(dst, src1, src2, cr);
14872 %}
14873 %}
14874
14875
14876 // ============================================================================
14877 // Branch Instructions
14878
14879 // Direct Branch.
14880 instruct branch(label lbl)
14881 %{
14882 match(Goto);
14883
14884 effect(USE lbl);
14885
14886 ins_cost(BRANCH_COST);
14887 format %{ "b $lbl" %}
14888
14889 ins_encode(aarch64_enc_b(lbl));
14890
14891 ins_pipe(pipe_branch);
14892 %}
14893
14894 // Conditional Near Branch
14895 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14896 %{
14897 // Same match rule as `branchConFar'.
14898 match(If cmp cr);
14899
14900 effect(USE lbl);
14901
14902 ins_cost(BRANCH_COST);
14903 // If set to 1 this indicates that the current instruction is a
14904 // short variant of a long branch. This avoids using this
14905 // instruction in first-pass matching. It will then only be used in
14906 // the `Shorten_branches' pass.
14907 // ins_short_branch(1);
14908 format %{ "b$cmp $lbl" %}
14909
14910 ins_encode(aarch64_enc_br_con(cmp, lbl));
14911
14912 ins_pipe(pipe_branch_cond);
14913 %}
14914
14915 // Conditional Near Branch Unsigned
14916 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14917 %{
14918 // Same match rule as `branchConFar'.
14919 match(If cmp cr);
14920
14921 effect(USE lbl);
14922
14923 ins_cost(BRANCH_COST);
14924 // If set to 1 this indicates that the current instruction is a
14925 // short variant of a long branch. This avoids using this
14926 // instruction in first-pass matching. It will then only be used in
14927 // the `Shorten_branches' pass.
14928 // ins_short_branch(1);
14929 format %{ "b$cmp $lbl\t# unsigned" %}
14930
14931 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14932
14933 ins_pipe(pipe_branch_cond);
14934 %}
14935
14936 // Make use of CBZ and CBNZ. These instructions, as well as being
14937 // shorter than (cmp; branch), have the additional benefit of not
14938 // killing the flags.
14939
14940 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14941 match(If cmp (CmpI op1 op2));
14942 effect(USE labl);
14943
14944 ins_cost(BRANCH_COST);
14945 format %{ "cbw$cmp $op1, $labl" %}
14946 ins_encode %{
14947 Label* L = $labl$$label;
14948 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14949 if (cond == Assembler::EQ)
14950 __ cbzw($op1$$Register, *L);
14951 else
14952 __ cbnzw($op1$$Register, *L);
14953 %}
14954 ins_pipe(pipe_cmp_branch);
14955 %}
14956
14957 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14958 match(If cmp (CmpL op1 op2));
14959 effect(USE labl);
14960
14961 ins_cost(BRANCH_COST);
14962 format %{ "cb$cmp $op1, $labl" %}
14963 ins_encode %{
14964 Label* L = $labl$$label;
14965 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14966 if (cond == Assembler::EQ)
14967 __ cbz($op1$$Register, *L);
14968 else
14969 __ cbnz($op1$$Register, *L);
14970 %}
14971 ins_pipe(pipe_cmp_branch);
14972 %}
14973
14974 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14975 match(If cmp (CmpP op1 op2));
14976 effect(USE labl);
14977
14978 ins_cost(BRANCH_COST);
14979 format %{ "cb$cmp $op1, $labl" %}
14980 ins_encode %{
14981 Label* L = $labl$$label;
14982 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14983 if (cond == Assembler::EQ)
14984 __ cbz($op1$$Register, *L);
14985 else
14986 __ cbnz($op1$$Register, *L);
14987 %}
14988 ins_pipe(pipe_cmp_branch);
14989 %}
14990
14991 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14992 match(If cmp (CmpN op1 op2));
14993 effect(USE labl);
14994
14995 ins_cost(BRANCH_COST);
14996 format %{ "cbw$cmp $op1, $labl" %}
14997 ins_encode %{
14998 Label* L = $labl$$label;
14999 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15000 if (cond == Assembler::EQ)
15001 __ cbzw($op1$$Register, *L);
15002 else
15003 __ cbnzw($op1$$Register, *L);
15004 %}
15005 ins_pipe(pipe_cmp_branch);
15006 %}
15007
15008 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15009 match(If cmp (CmpP (DecodeN oop) zero));
15010 effect(USE labl);
15011
15012 ins_cost(BRANCH_COST);
15013 format %{ "cb$cmp $oop, $labl" %}
15014 ins_encode %{
15015 Label* L = $labl$$label;
15016 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15017 if (cond == Assembler::EQ)
15018 __ cbzw($oop$$Register, *L);
15019 else
15020 __ cbnzw($oop$$Register, *L);
15021 %}
15022 ins_pipe(pipe_cmp_branch);
15023 %}
15024
15025 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15026 match(If cmp (CmpU op1 op2));
15027 effect(USE labl);
15028
15029 ins_cost(BRANCH_COST);
15030 format %{ "cbw$cmp $op1, $labl" %}
15031 ins_encode %{
15032 Label* L = $labl$$label;
15033 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15034 if (cond == Assembler::EQ || cond == Assembler::LS) {
15035 __ cbzw($op1$$Register, *L);
15036 } else {
15037 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15038 __ cbnzw($op1$$Register, *L);
15039 }
15040 %}
15041 ins_pipe(pipe_cmp_branch);
15042 %}
15043
15044 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15045 match(If cmp (CmpUL op1 op2));
15046 effect(USE labl);
15047
15048 ins_cost(BRANCH_COST);
15049 format %{ "cb$cmp $op1, $labl" %}
15050 ins_encode %{
15051 Label* L = $labl$$label;
15052 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15053 if (cond == Assembler::EQ || cond == Assembler::LS) {
15054 __ cbz($op1$$Register, *L);
15055 } else {
15056 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15057 __ cbnz($op1$$Register, *L);
15058 }
15059 %}
15060 ins_pipe(pipe_cmp_branch);
15061 %}
15062
15063 // Test bit and Branch
15064
15065 // Patterns for short (< 32KiB) variants
15066 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15067 match(If cmp (CmpL op1 op2));
15068 effect(USE labl);
15069
15070 ins_cost(BRANCH_COST);
15071 format %{ "cb$cmp $op1, $labl # long" %}
15072 ins_encode %{
15073 Label* L = $labl$$label;
15074 Assembler::Condition cond =
15075 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15076 __ tbr(cond, $op1$$Register, 63, *L);
15077 %}
15078 ins_pipe(pipe_cmp_branch);
15079 ins_short_branch(1);
15080 %}
15081
15082 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15083 match(If cmp (CmpI op1 op2));
15084 effect(USE labl);
15085
15086 ins_cost(BRANCH_COST);
15087 format %{ "cb$cmp $op1, $labl # int" %}
15088 ins_encode %{
15089 Label* L = $labl$$label;
15090 Assembler::Condition cond =
15091 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15092 __ tbr(cond, $op1$$Register, 31, *L);
15093 %}
15094 ins_pipe(pipe_cmp_branch);
15095 ins_short_branch(1);
15096 %}
15097
15098 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15099 match(If cmp (CmpL (AndL op1 op2) op3));
15100 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15101 effect(USE labl);
15102
15103 ins_cost(BRANCH_COST);
15104 format %{ "tb$cmp $op1, $op2, $labl" %}
15105 ins_encode %{
15106 Label* L = $labl$$label;
15107 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15108 int bit = exact_log2_long($op2$$constant);
15109 __ tbr(cond, $op1$$Register, bit, *L);
15110 %}
15111 ins_pipe(pipe_cmp_branch);
15112 ins_short_branch(1);
15113 %}
15114
15115 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15116 match(If cmp (CmpI (AndI op1 op2) op3));
15117 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15118 effect(USE labl);
15119
15120 ins_cost(BRANCH_COST);
15121 format %{ "tb$cmp $op1, $op2, $labl" %}
15122 ins_encode %{
15123 Label* L = $labl$$label;
15124 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15125 int bit = exact_log2((juint)$op2$$constant);
15126 __ tbr(cond, $op1$$Register, bit, *L);
15127 %}
15128 ins_pipe(pipe_cmp_branch);
15129 ins_short_branch(1);
15130 %}
15131
15132 // And far variants
15133 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15134 match(If cmp (CmpL op1 op2));
15135 effect(USE labl);
15136
15137 ins_cost(BRANCH_COST);
15138 format %{ "cb$cmp $op1, $labl # long" %}
15139 ins_encode %{
15140 Label* L = $labl$$label;
15141 Assembler::Condition cond =
15142 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15143 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15144 %}
15145 ins_pipe(pipe_cmp_branch);
15146 %}
15147
15148 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15149 match(If cmp (CmpI op1 op2));
15150 effect(USE labl);
15151
15152 ins_cost(BRANCH_COST);
15153 format %{ "cb$cmp $op1, $labl # int" %}
15154 ins_encode %{
15155 Label* L = $labl$$label;
15156 Assembler::Condition cond =
15157 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15158 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15159 %}
15160 ins_pipe(pipe_cmp_branch);
15161 %}
15162
15163 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15164 match(If cmp (CmpL (AndL op1 op2) op3));
15165 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15166 effect(USE labl);
15167
15168 ins_cost(BRANCH_COST);
15169 format %{ "tb$cmp $op1, $op2, $labl" %}
15170 ins_encode %{
15171 Label* L = $labl$$label;
15172 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15173 int bit = exact_log2_long($op2$$constant);
15174 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15175 %}
15176 ins_pipe(pipe_cmp_branch);
15177 %}
15178
15179 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15180 match(If cmp (CmpI (AndI op1 op2) op3));
15181 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15182 effect(USE labl);
15183
15184 ins_cost(BRANCH_COST);
15185 format %{ "tb$cmp $op1, $op2, $labl" %}
15186 ins_encode %{
15187 Label* L = $labl$$label;
15188 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15189 int bit = exact_log2((juint)$op2$$constant);
15190 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15191 %}
15192 ins_pipe(pipe_cmp_branch);
15193 %}
15194
15195 // Test bits
15196
15197 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15198 match(Set cr (CmpL (AndL op1 op2) op3));
15199 predicate(Assembler::operand_valid_for_logical_immediate
15200 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15201
15202 ins_cost(INSN_COST);
15203 format %{ "tst $op1, $op2 # long" %}
15204 ins_encode %{
15205 __ tst($op1$$Register, $op2$$constant);
15206 %}
15207 ins_pipe(ialu_reg_reg);
15208 %}
15209
15210 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15211 match(Set cr (CmpI (AndI op1 op2) op3));
15212 predicate(Assembler::operand_valid_for_logical_immediate
15213 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15214
15215 ins_cost(INSN_COST);
15216 format %{ "tst $op1, $op2 # int" %}
15217 ins_encode %{
15218 __ tstw($op1$$Register, $op2$$constant);
15219 %}
15220 ins_pipe(ialu_reg_reg);
15221 %}
15222
15223 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15224 match(Set cr (CmpL (AndL op1 op2) op3));
15225
15226 ins_cost(INSN_COST);
15227 format %{ "tst $op1, $op2 # long" %}
15228 ins_encode %{
15229 __ tst($op1$$Register, $op2$$Register);
15230 %}
15231 ins_pipe(ialu_reg_reg);
15232 %}
15233
15234 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15235 match(Set cr (CmpI (AndI op1 op2) op3));
15236
15237 ins_cost(INSN_COST);
15238 format %{ "tstw $op1, $op2 # int" %}
15239 ins_encode %{
15240 __ tstw($op1$$Register, $op2$$Register);
15241 %}
15242 ins_pipe(ialu_reg_reg);
15243 %}
15244
15245
15246 // Conditional Far Branch
15247 // Conditional Far Branch Unsigned
15248 // TODO: fixme
15249
15250 // counted loop end branch near
15251 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15252 %{
15253 match(CountedLoopEnd cmp cr);
15254
15255 effect(USE lbl);
15256
15257 ins_cost(BRANCH_COST);
15258 // short variant.
15259 // ins_short_branch(1);
15260 format %{ "b$cmp $lbl \t// counted loop end" %}
15261
15262 ins_encode(aarch64_enc_br_con(cmp, lbl));
15263
15264 ins_pipe(pipe_branch);
15265 %}
15266
15267 // counted loop end branch far
15268 // TODO: fixme
15269
15270 // ============================================================================
15271 // inlined locking and unlocking
15272
15273 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15274 %{
15275 match(Set cr (FastLock object box));
15276 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15277
15278 ins_cost(5 * INSN_COST);
15279 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15280
15281 ins_encode %{
15282 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15283 %}
15284
15285 ins_pipe(pipe_serial);
15286 %}
15287
15288 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15289 %{
15290 match(Set cr (FastUnlock object box));
15291 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15292
15293 ins_cost(5 * INSN_COST);
15294 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15295
15296 ins_encode %{
15297 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15298 %}
15299
15300 ins_pipe(pipe_serial);
15301 %}
15302
15303 // ============================================================================
15304 // Safepoint Instructions
15305
15306 // TODO
15307 // provide a near and far version of this code
15308
15309 instruct safePoint(rFlagsReg cr, iRegP poll)
15310 %{
15311 match(SafePoint poll);
15312 effect(KILL cr);
15313
15314 format %{
15315 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15316 %}
15317 ins_encode %{
15318 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15319 %}
15320 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15321 %}
15322
15323
15324 // ============================================================================
15325 // Procedure Call/Return Instructions
15326
15327 // Call Java Static Instruction
15328
15329 instruct CallStaticJavaDirect(method meth)
15330 %{
15331 match(CallStaticJava);
15332
15333 effect(USE meth);
15334
15335 ins_cost(CALL_COST);
15336
15337 format %{ "call,static $meth \t// ==> " %}
15338
15339 ins_encode(aarch64_enc_java_static_call(meth),
15340 aarch64_enc_call_epilog);
15341
15342 ins_pipe(pipe_class_call);
15343 %}
15344
15345 // TO HERE
15346
15347 // Call Java Dynamic Instruction
15348 instruct CallDynamicJavaDirect(method meth)
15349 %{
15350 match(CallDynamicJava);
15351
15352 effect(USE meth);
15353
15354 ins_cost(CALL_COST);
15355
15356 format %{ "CALL,dynamic $meth \t// ==> " %}
15357
15358 ins_encode(aarch64_enc_java_dynamic_call(meth),
15359 aarch64_enc_call_epilog);
15360
15361 ins_pipe(pipe_class_call);
15362 %}
15363
15364 // Call Runtime Instruction
15365
15366 instruct CallRuntimeDirect(method meth)
15367 %{
15368 match(CallRuntime);
15369
15370 effect(USE meth);
15371
15372 ins_cost(CALL_COST);
15373
15374 format %{ "CALL, runtime $meth" %}
15375
15376 ins_encode( aarch64_enc_java_to_runtime(meth) );
15377
15378 ins_pipe(pipe_class_call);
15379 %}
15380
15381 // Call Runtime Instruction
15382
15383 instruct CallLeafDirect(method meth)
15384 %{
15385 match(CallLeaf);
15386
15387 effect(USE meth);
15388
15389 ins_cost(CALL_COST);
15390
15391 format %{ "CALL, runtime leaf $meth" %}
15392
15393 ins_encode( aarch64_enc_java_to_runtime(meth) );
15394
15395 ins_pipe(pipe_class_call);
15396 %}
15397
15398 // Call Runtime Instruction without safepoint and with vector arguments
15399 instruct CallLeafDirectVector(method meth)
15400 %{
15401 match(CallLeafVector);
15402
15403 effect(USE meth);
15404
15405 ins_cost(CALL_COST);
15406
15407 format %{ "CALL, runtime leaf vector $meth" %}
15408
15409 ins_encode(aarch64_enc_java_to_runtime(meth));
15410
15411 ins_pipe(pipe_class_call);
15412 %}
15413
15414 // Call Runtime Instruction
15415
15416 instruct CallLeafNoFPDirect(method meth)
15417 %{
15418 match(CallLeafNoFP);
15419
15420 effect(USE meth);
15421
15422 ins_cost(CALL_COST);
15423
15424 format %{ "CALL, runtime leaf nofp $meth" %}
15425
15426 ins_encode( aarch64_enc_java_to_runtime(meth) );
15427
15428 ins_pipe(pipe_class_call);
15429 %}
15430
15431 // Tail Call; Jump from runtime stub to Java code.
15432 // Also known as an 'interprocedural jump'.
15433 // Target of jump will eventually return to caller.
15434 // TailJump below removes the return address.
15435 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15436 // emitted just above the TailCall which has reset rfp to the caller state.
15437 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15438 %{
15439 match(TailCall jump_target method_ptr);
15440
15441 ins_cost(CALL_COST);
15442
15443 format %{ "br $jump_target\t# $method_ptr holds method" %}
15444
15445 ins_encode(aarch64_enc_tail_call(jump_target));
15446
15447 ins_pipe(pipe_class_call);
15448 %}
15449
15450 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15451 %{
15452 match(TailJump jump_target ex_oop);
15453
15454 ins_cost(CALL_COST);
15455
15456 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15457
15458 ins_encode(aarch64_enc_tail_jmp(jump_target));
15459
15460 ins_pipe(pipe_class_call);
15461 %}
15462
15463 // Forward exception.
15464 instruct ForwardExceptionjmp()
15465 %{
15466 match(ForwardException);
15467 ins_cost(CALL_COST);
15468
15469 format %{ "b forward_exception_stub" %}
15470 ins_encode %{
15471 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15472 %}
15473 ins_pipe(pipe_class_call);
15474 %}
15475
15476 // Create exception oop: created by stack-crawling runtime code.
15477 // Created exception is now available to this handler, and is setup
15478 // just prior to jumping to this handler. No code emitted.
15479 // TODO check
15480 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15481 instruct CreateException(iRegP_R0 ex_oop)
15482 %{
15483 match(Set ex_oop (CreateEx));
15484
15485 format %{ " -- \t// exception oop; no code emitted" %}
15486
15487 size(0);
15488
15489 ins_encode( /*empty*/ );
15490
15491 ins_pipe(pipe_class_empty);
15492 %}
15493
15494 // Rethrow exception: The exception oop will come in the first
15495 // argument position. Then JUMP (not call) to the rethrow stub code.
15496 instruct RethrowException() %{
15497 match(Rethrow);
15498 ins_cost(CALL_COST);
15499
15500 format %{ "b rethrow_stub" %}
15501
15502 ins_encode( aarch64_enc_rethrow() );
15503
15504 ins_pipe(pipe_class_call);
15505 %}
15506
15507
15508 // Return Instruction
15509 // epilog node loads ret address into lr as part of frame pop
15510 instruct Ret()
15511 %{
15512 match(Return);
15513
15514 format %{ "ret\t// return register" %}
15515
15516 ins_encode( aarch64_enc_ret() );
15517
15518 ins_pipe(pipe_branch);
15519 %}
15520
15521 // Die now.
15522 instruct ShouldNotReachHere() %{
15523 match(Halt);
15524
15525 ins_cost(CALL_COST);
15526 format %{ "ShouldNotReachHere" %}
15527
15528 ins_encode %{
15529 if (is_reachable()) {
15530 const char* str = __ code_string(_halt_reason);
15531 __ stop(str);
15532 }
15533 %}
15534
15535 ins_pipe(pipe_class_default);
15536 %}
15537
15538 // ============================================================================
15539 // Partial Subtype Check
15540 //
15541 // superklass array for an instance of the superklass. Set a hidden
15542 // internal cache on a hit (cache is checked with exposed code in
15543 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15544 // encoding ALSO sets flags.
15545
15546 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15547 %{
15548 match(Set result (PartialSubtypeCheck sub super));
15549 predicate(!UseSecondarySupersTable);
15550 effect(KILL cr, KILL temp);
15551
15552 ins_cost(20 * INSN_COST); // slightly larger than the next version
15553 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15554
15555 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15556
15557 opcode(0x1); // Force zero of result reg on hit
15558
15559 ins_pipe(pipe_class_memory);
15560 %}
15561
15562 // Two versions of partialSubtypeCheck, both used when we need to
15563 // search for a super class in the secondary supers array. The first
15564 // is used when we don't know _a priori_ the class being searched
15565 // for. The second, far more common, is used when we do know: this is
15566 // used for instanceof, checkcast, and any case where C2 can determine
15567 // it by constant propagation.
15568
15569 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15570 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15571 rFlagsReg cr)
15572 %{
15573 match(Set result (PartialSubtypeCheck sub super));
15574 predicate(UseSecondarySupersTable);
15575 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15576
15577 ins_cost(10 * INSN_COST); // slightly larger than the next version
15578 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15579
15580 ins_encode %{
15581 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15582 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15583 $vtemp$$FloatRegister,
15584 $result$$Register, /*L_success*/nullptr);
15585 %}
15586
15587 ins_pipe(pipe_class_memory);
15588 %}
15589
15590 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15591 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15592 rFlagsReg cr)
15593 %{
15594 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15595 predicate(UseSecondarySupersTable);
15596 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15597
15598 ins_cost(5 * INSN_COST); // smaller than the next version
15599 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15600
15601 ins_encode %{
15602 bool success = false;
15603 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15604 if (InlineSecondarySupersTest) {
15605 success =
15606 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15607 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15608 $vtemp$$FloatRegister,
15609 $result$$Register,
15610 super_klass_slot);
15611 } else {
15612 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15613 success = (call != nullptr);
15614 }
15615 if (!success) {
15616 ciEnv::current()->record_failure("CodeCache is full");
15617 return;
15618 }
15619 %}
15620
15621 ins_pipe(pipe_class_memory);
15622 %}
15623
15624 // Intrisics for String.compareTo()
15625
15626 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15627 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15628 %{
15629 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15630 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15631 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15632
15633 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15634 ins_encode %{
15635 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15636 __ string_compare($str1$$Register, $str2$$Register,
15637 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15638 $tmp1$$Register, $tmp2$$Register,
15639 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15640 %}
15641 ins_pipe(pipe_class_memory);
15642 %}
15643
15644 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15645 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15646 %{
15647 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15648 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15649 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15650
15651 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15652 ins_encode %{
15653 __ string_compare($str1$$Register, $str2$$Register,
15654 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15655 $tmp1$$Register, $tmp2$$Register,
15656 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15657 %}
15658 ins_pipe(pipe_class_memory);
15659 %}
15660
15661 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15662 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15663 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15664 %{
15665 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15666 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15667 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15668 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15669
15670 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15671 ins_encode %{
15672 __ string_compare($str1$$Register, $str2$$Register,
15673 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15674 $tmp1$$Register, $tmp2$$Register,
15675 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15676 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15677 %}
15678 ins_pipe(pipe_class_memory);
15679 %}
15680
15681 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15682 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15683 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15684 %{
15685 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15686 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15687 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15688 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15689
15690 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15691 ins_encode %{
15692 __ string_compare($str1$$Register, $str2$$Register,
15693 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15694 $tmp1$$Register, $tmp2$$Register,
15695 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15696 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15697 %}
15698 ins_pipe(pipe_class_memory);
15699 %}
15700
15701 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15702 // these string_compare variants as NEON register type for convenience so that the prototype of
15703 // string_compare can be shared with all variants.
15704
15705 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15706 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15707 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15708 pRegGov_P1 pgtmp2, rFlagsReg cr)
15709 %{
15710 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15711 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15712 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15713 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15714
15715 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15716 ins_encode %{
15717 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15718 __ string_compare($str1$$Register, $str2$$Register,
15719 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15720 $tmp1$$Register, $tmp2$$Register,
15721 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15722 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15723 StrIntrinsicNode::LL);
15724 %}
15725 ins_pipe(pipe_class_memory);
15726 %}
15727
15728 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15729 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15730 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15731 pRegGov_P1 pgtmp2, rFlagsReg cr)
15732 %{
15733 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15734 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15735 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15736 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15737
15738 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15739 ins_encode %{
15740 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15741 __ string_compare($str1$$Register, $str2$$Register,
15742 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15743 $tmp1$$Register, $tmp2$$Register,
15744 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15745 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15746 StrIntrinsicNode::LU);
15747 %}
15748 ins_pipe(pipe_class_memory);
15749 %}
15750
15751 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15752 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15753 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15754 pRegGov_P1 pgtmp2, rFlagsReg cr)
15755 %{
15756 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15757 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15758 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15759 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15760
15761 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15762 ins_encode %{
15763 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15764 __ string_compare($str1$$Register, $str2$$Register,
15765 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15766 $tmp1$$Register, $tmp2$$Register,
15767 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15768 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15769 StrIntrinsicNode::UL);
15770 %}
15771 ins_pipe(pipe_class_memory);
15772 %}
15773
15774 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15775 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15776 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15777 pRegGov_P1 pgtmp2, rFlagsReg cr)
15778 %{
15779 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15780 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15781 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15782 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15783
15784 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15785 ins_encode %{
15786 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15787 __ string_compare($str1$$Register, $str2$$Register,
15788 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15789 $tmp1$$Register, $tmp2$$Register,
15790 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15791 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15792 StrIntrinsicNode::UU);
15793 %}
15794 ins_pipe(pipe_class_memory);
15795 %}
15796
15797 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15798 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15799 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15800 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15801 %{
15802 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15803 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15804 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15805 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15806 TEMP vtmp0, TEMP vtmp1, KILL cr);
15807 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15808 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15809
15810 ins_encode %{
15811 __ string_indexof($str1$$Register, $str2$$Register,
15812 $cnt1$$Register, $cnt2$$Register,
15813 $tmp1$$Register, $tmp2$$Register,
15814 $tmp3$$Register, $tmp4$$Register,
15815 $tmp5$$Register, $tmp6$$Register,
15816 -1, $result$$Register, StrIntrinsicNode::UU);
15817 %}
15818 ins_pipe(pipe_class_memory);
15819 %}
15820
15821 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15822 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15823 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15824 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15825 %{
15826 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15827 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15828 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15829 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15830 TEMP vtmp0, TEMP vtmp1, KILL cr);
15831 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15832 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15833
15834 ins_encode %{
15835 __ string_indexof($str1$$Register, $str2$$Register,
15836 $cnt1$$Register, $cnt2$$Register,
15837 $tmp1$$Register, $tmp2$$Register,
15838 $tmp3$$Register, $tmp4$$Register,
15839 $tmp5$$Register, $tmp6$$Register,
15840 -1, $result$$Register, StrIntrinsicNode::LL);
15841 %}
15842 ins_pipe(pipe_class_memory);
15843 %}
15844
15845 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15846 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15847 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15848 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15849 %{
15850 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15851 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15852 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15853 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15854 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15855 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15856 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15857
15858 ins_encode %{
15859 __ string_indexof($str1$$Register, $str2$$Register,
15860 $cnt1$$Register, $cnt2$$Register,
15861 $tmp1$$Register, $tmp2$$Register,
15862 $tmp3$$Register, $tmp4$$Register,
15863 $tmp5$$Register, $tmp6$$Register,
15864 -1, $result$$Register, StrIntrinsicNode::UL);
15865 %}
15866 ins_pipe(pipe_class_memory);
15867 %}
15868
15869 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15870 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15871 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15872 %{
15873 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15874 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15875 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15876 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15877 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15878 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15879
15880 ins_encode %{
15881 int icnt2 = (int)$int_cnt2$$constant;
15882 __ string_indexof($str1$$Register, $str2$$Register,
15883 $cnt1$$Register, zr,
15884 $tmp1$$Register, $tmp2$$Register,
15885 $tmp3$$Register, $tmp4$$Register, zr, zr,
15886 icnt2, $result$$Register, StrIntrinsicNode::UU);
15887 %}
15888 ins_pipe(pipe_class_memory);
15889 %}
15890
15891 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15892 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15893 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15894 %{
15895 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15896 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15897 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15898 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15899 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15900 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15901
15902 ins_encode %{
15903 int icnt2 = (int)$int_cnt2$$constant;
15904 __ string_indexof($str1$$Register, $str2$$Register,
15905 $cnt1$$Register, zr,
15906 $tmp1$$Register, $tmp2$$Register,
15907 $tmp3$$Register, $tmp4$$Register, zr, zr,
15908 icnt2, $result$$Register, StrIntrinsicNode::LL);
15909 %}
15910 ins_pipe(pipe_class_memory);
15911 %}
15912
15913 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15914 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15915 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15916 %{
15917 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15918 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15919 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15920 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15921 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15922 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15923
15924 ins_encode %{
15925 int icnt2 = (int)$int_cnt2$$constant;
15926 __ string_indexof($str1$$Register, $str2$$Register,
15927 $cnt1$$Register, zr,
15928 $tmp1$$Register, $tmp2$$Register,
15929 $tmp3$$Register, $tmp4$$Register, zr, zr,
15930 icnt2, $result$$Register, StrIntrinsicNode::UL);
15931 %}
15932 ins_pipe(pipe_class_memory);
15933 %}
15934
15935 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15936 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15937 iRegINoSp tmp3, rFlagsReg cr)
15938 %{
15939 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15940 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15941 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15942 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15943
15944 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15945
15946 ins_encode %{
15947 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15948 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15949 $tmp3$$Register);
15950 %}
15951 ins_pipe(pipe_class_memory);
15952 %}
15953
15954 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15955 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15956 iRegINoSp tmp3, rFlagsReg cr)
15957 %{
15958 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15959 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15960 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15961 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15962
15963 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15964
15965 ins_encode %{
15966 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15967 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15968 $tmp3$$Register);
15969 %}
15970 ins_pipe(pipe_class_memory);
15971 %}
15972
15973 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15974 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15975 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15976 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15977 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15978 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15979 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15980 ins_encode %{
15981 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15982 $result$$Register, $ztmp1$$FloatRegister,
15983 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15984 $ptmp$$PRegister, true /* isL */);
15985 %}
15986 ins_pipe(pipe_class_memory);
15987 %}
15988
15989 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15990 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15991 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15992 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15993 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15994 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15995 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15996 ins_encode %{
15997 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15998 $result$$Register, $ztmp1$$FloatRegister,
15999 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16000 $ptmp$$PRegister, false /* isL */);
16001 %}
16002 ins_pipe(pipe_class_memory);
16003 %}
16004
16005 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16006 iRegI_R0 result, rFlagsReg cr)
16007 %{
16008 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16009 match(Set result (StrEquals (Binary str1 str2) cnt));
16010 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16011
16012 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16013 ins_encode %{
16014 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16015 __ string_equals($str1$$Register, $str2$$Register,
16016 $result$$Register, $cnt$$Register);
16017 %}
16018 ins_pipe(pipe_class_memory);
16019 %}
16020
16021 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16022 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16023 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16024 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16025 iRegP_R10 tmp, rFlagsReg cr)
16026 %{
16027 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16028 match(Set result (AryEq ary1 ary2));
16029 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16030 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16031 TEMP vtmp6, TEMP vtmp7, KILL cr);
16032
16033 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16034 ins_encode %{
16035 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16036 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16037 $result$$Register, $tmp$$Register, 1);
16038 if (tpc == nullptr) {
16039 ciEnv::current()->record_failure("CodeCache is full");
16040 return;
16041 }
16042 %}
16043 ins_pipe(pipe_class_memory);
16044 %}
16045
16046 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16047 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16048 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16049 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16050 iRegP_R10 tmp, rFlagsReg cr)
16051 %{
16052 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16053 match(Set result (AryEq ary1 ary2));
16054 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16055 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16056 TEMP vtmp6, TEMP vtmp7, KILL cr);
16057
16058 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16059 ins_encode %{
16060 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16061 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16062 $result$$Register, $tmp$$Register, 2);
16063 if (tpc == nullptr) {
16064 ciEnv::current()->record_failure("CodeCache is full");
16065 return;
16066 }
16067 %}
16068 ins_pipe(pipe_class_memory);
16069 %}
16070
16071 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16072 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16073 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16074 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16075 %{
16076 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16077 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16078 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16079
16080 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16081 ins_encode %{
16082 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16083 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16084 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16085 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16086 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16087 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16088 (BasicType)$basic_type$$constant);
16089 if (tpc == nullptr) {
16090 ciEnv::current()->record_failure("CodeCache is full");
16091 return;
16092 }
16093 %}
16094 ins_pipe(pipe_class_memory);
16095 %}
16096
16097 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16098 %{
16099 match(Set result (CountPositives ary1 len));
16100 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16101 format %{ "count positives byte[] $ary1,$len -> $result" %}
16102 ins_encode %{
16103 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16104 if (tpc == nullptr) {
16105 ciEnv::current()->record_failure("CodeCache is full");
16106 return;
16107 }
16108 %}
16109 ins_pipe( pipe_slow );
16110 %}
16111
16112 // fast char[] to byte[] compression
16113 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16114 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16115 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16116 iRegI_R0 result, rFlagsReg cr)
16117 %{
16118 match(Set result (StrCompressedCopy src (Binary dst len)));
16119 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16120 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16121
16122 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16123 ins_encode %{
16124 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16125 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16126 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16127 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16128 %}
16129 ins_pipe(pipe_slow);
16130 %}
16131
16132 // fast byte[] to char[] inflation
16133 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16134 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16135 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16136 %{
16137 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16138 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16139 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16140 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16141
16142 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16143 ins_encode %{
16144 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16145 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16146 $vtmp2$$FloatRegister, $tmp$$Register);
16147 if (tpc == nullptr) {
16148 ciEnv::current()->record_failure("CodeCache is full");
16149 return;
16150 }
16151 %}
16152 ins_pipe(pipe_class_memory);
16153 %}
16154
16155 // encode char[] to byte[] in ISO_8859_1
16156 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16157 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16158 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16159 iRegI_R0 result, rFlagsReg cr)
16160 %{
16161 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16162 match(Set result (EncodeISOArray src (Binary dst len)));
16163 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16164 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16165
16166 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16167 ins_encode %{
16168 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16169 $result$$Register, false,
16170 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16171 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16172 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16173 %}
16174 ins_pipe(pipe_class_memory);
16175 %}
16176
16177 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16178 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16179 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16180 iRegI_R0 result, rFlagsReg cr)
16181 %{
16182 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16183 match(Set result (EncodeISOArray src (Binary dst len)));
16184 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16185 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16186
16187 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16188 ins_encode %{
16189 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16190 $result$$Register, true,
16191 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16192 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16193 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16194 %}
16195 ins_pipe(pipe_class_memory);
16196 %}
16197
16198 //----------------------------- CompressBits/ExpandBits ------------------------
16199
16200 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16201 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16202 match(Set dst (CompressBits src mask));
16203 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16204 format %{ "mov $tsrc, $src\n\t"
16205 "mov $tmask, $mask\n\t"
16206 "bext $tdst, $tsrc, $tmask\n\t"
16207 "mov $dst, $tdst"
16208 %}
16209 ins_encode %{
16210 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16211 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16212 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16213 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16214 %}
16215 ins_pipe(pipe_slow);
16216 %}
16217
16218 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16219 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16220 match(Set dst (CompressBits (LoadI mem) mask));
16221 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16222 format %{ "ldrs $tsrc, $mem\n\t"
16223 "ldrs $tmask, $mask\n\t"
16224 "bext $tdst, $tsrc, $tmask\n\t"
16225 "mov $dst, $tdst"
16226 %}
16227 ins_encode %{
16228 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16229 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16230 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16231 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16232 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16233 %}
16234 ins_pipe(pipe_slow);
16235 %}
16236
16237 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16238 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16239 match(Set dst (CompressBits src mask));
16240 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16241 format %{ "mov $tsrc, $src\n\t"
16242 "mov $tmask, $mask\n\t"
16243 "bext $tdst, $tsrc, $tmask\n\t"
16244 "mov $dst, $tdst"
16245 %}
16246 ins_encode %{
16247 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16248 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16249 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16250 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16251 %}
16252 ins_pipe(pipe_slow);
16253 %}
16254
16255 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16256 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16257 match(Set dst (CompressBits (LoadL mem) mask));
16258 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16259 format %{ "ldrd $tsrc, $mem\n\t"
16260 "ldrd $tmask, $mask\n\t"
16261 "bext $tdst, $tsrc, $tmask\n\t"
16262 "mov $dst, $tdst"
16263 %}
16264 ins_encode %{
16265 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16266 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16267 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16268 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16269 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16270 %}
16271 ins_pipe(pipe_slow);
16272 %}
16273
16274 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16275 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16276 match(Set dst (ExpandBits src mask));
16277 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16278 format %{ "mov $tsrc, $src\n\t"
16279 "mov $tmask, $mask\n\t"
16280 "bdep $tdst, $tsrc, $tmask\n\t"
16281 "mov $dst, $tdst"
16282 %}
16283 ins_encode %{
16284 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16285 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16286 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16287 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16288 %}
16289 ins_pipe(pipe_slow);
16290 %}
16291
16292 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16293 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16294 match(Set dst (ExpandBits (LoadI mem) mask));
16295 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16296 format %{ "ldrs $tsrc, $mem\n\t"
16297 "ldrs $tmask, $mask\n\t"
16298 "bdep $tdst, $tsrc, $tmask\n\t"
16299 "mov $dst, $tdst"
16300 %}
16301 ins_encode %{
16302 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16303 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16304 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16305 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16306 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16307 %}
16308 ins_pipe(pipe_slow);
16309 %}
16310
16311 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16312 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16313 match(Set dst (ExpandBits src mask));
16314 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16315 format %{ "mov $tsrc, $src\n\t"
16316 "mov $tmask, $mask\n\t"
16317 "bdep $tdst, $tsrc, $tmask\n\t"
16318 "mov $dst, $tdst"
16319 %}
16320 ins_encode %{
16321 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16322 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16323 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16324 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16325 %}
16326 ins_pipe(pipe_slow);
16327 %}
16328
16329
16330 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16331 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16332 match(Set dst (ExpandBits (LoadL mem) mask));
16333 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16334 format %{ "ldrd $tsrc, $mem\n\t"
16335 "ldrd $tmask, $mask\n\t"
16336 "bdep $tdst, $tsrc, $tmask\n\t"
16337 "mov $dst, $tdst"
16338 %}
16339 ins_encode %{
16340 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16341 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16342 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16343 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16344 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16345 %}
16346 ins_pipe(pipe_slow);
16347 %}
16348
16349 //----------------------------- Reinterpret ----------------------------------
16350 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16351 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16352 match(Set dst (ReinterpretHF2S src));
16353 format %{ "reinterpretHF2S $dst, $src" %}
16354 ins_encode %{
16355 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16356 %}
16357 ins_pipe(pipe_slow);
16358 %}
16359
16360 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16361 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16362 match(Set dst (ReinterpretS2HF src));
16363 format %{ "reinterpretS2HF $dst, $src" %}
16364 ins_encode %{
16365 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16366 %}
16367 ins_pipe(pipe_slow);
16368 %}
16369
16370 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16371 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16372 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16373 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16374 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16375 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16376 // can be omitted in this pattern, resulting in -
16377 // fcvt $dst, $src // Convert float to half-precision float
16378 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16379 %{
16380 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16381 format %{ "convF2HFAndS2HF $dst, $src" %}
16382 ins_encode %{
16383 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16384 %}
16385 ins_pipe(pipe_slow);
16386 %}
16387
16388 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16389 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16390 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16391 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16392 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16393 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16394 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16395 // resulting in -
16396 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16397 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16398 %{
16399 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16400 format %{ "convHF2SAndHF2F $dst, $src" %}
16401 ins_encode %{
16402 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16403 %}
16404 ins_pipe(pipe_slow);
16405 %}
16406
16407 // ============================================================================
16408 // This name is KNOWN by the ADLC and cannot be changed.
16409 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16410 // for this guy.
16411 instruct tlsLoadP(thread_RegP dst)
16412 %{
16413 match(Set dst (ThreadLocal));
16414
16415 ins_cost(0);
16416
16417 format %{ " -- \t// $dst=Thread::current(), empty" %}
16418
16419 size(0);
16420
16421 ins_encode( /*empty*/ );
16422
16423 ins_pipe(pipe_class_empty);
16424 %}
16425
16426 //----------PEEPHOLE RULES-----------------------------------------------------
16427 // These must follow all instruction definitions as they use the names
16428 // defined in the instructions definitions.
16429 //
16430 // peepmatch ( root_instr_name [preceding_instruction]* );
16431 //
16432 // peepconstraint %{
16433 // (instruction_number.operand_name relational_op instruction_number.operand_name
16434 // [, ...] );
16435 // // instruction numbers are zero-based using left to right order in peepmatch
16436 //
16437 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16438 // // provide an instruction_number.operand_name for each operand that appears
16439 // // in the replacement instruction's match rule
16440 //
16441 // ---------VM FLAGS---------------------------------------------------------
16442 //
16443 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16444 //
16445 // Each peephole rule is given an identifying number starting with zero and
16446 // increasing by one in the order seen by the parser. An individual peephole
16447 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16448 // on the command-line.
16449 //
16450 // ---------CURRENT LIMITATIONS----------------------------------------------
16451 //
16452 // Only match adjacent instructions in same basic block
16453 // Only equality constraints
16454 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16455 // Only one replacement instruction
16456 //
16457 // ---------EXAMPLE----------------------------------------------------------
16458 //
16459 // // pertinent parts of existing instructions in architecture description
16460 // instruct movI(iRegINoSp dst, iRegI src)
16461 // %{
16462 // match(Set dst (CopyI src));
16463 // %}
16464 //
16465 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16466 // %{
16467 // match(Set dst (AddI dst src));
16468 // effect(KILL cr);
16469 // %}
16470 //
16471 // // Change (inc mov) to lea
16472 // peephole %{
16473 // // increment preceded by register-register move
16474 // peepmatch ( incI_iReg movI );
16475 // // require that the destination register of the increment
16476 // // match the destination register of the move
16477 // peepconstraint ( 0.dst == 1.dst );
16478 // // construct a replacement instruction that sets
16479 // // the destination to ( move's source register + one )
16480 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16481 // %}
16482 //
16483
16484 // Implementation no longer uses movX instructions since
16485 // machine-independent system no longer uses CopyX nodes.
16486 //
16487 // peephole
16488 // %{
16489 // peepmatch (incI_iReg movI);
16490 // peepconstraint (0.dst == 1.dst);
16491 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16492 // %}
16493
16494 // peephole
16495 // %{
16496 // peepmatch (decI_iReg movI);
16497 // peepconstraint (0.dst == 1.dst);
16498 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16499 // %}
16500
16501 // peephole
16502 // %{
16503 // peepmatch (addI_iReg_imm movI);
16504 // peepconstraint (0.dst == 1.dst);
16505 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16506 // %}
16507
16508 // peephole
16509 // %{
16510 // peepmatch (incL_iReg movL);
16511 // peepconstraint (0.dst == 1.dst);
16512 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16513 // %}
16514
16515 // peephole
16516 // %{
16517 // peepmatch (decL_iReg movL);
16518 // peepconstraint (0.dst == 1.dst);
16519 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16520 // %}
16521
16522 // peephole
16523 // %{
16524 // peepmatch (addL_iReg_imm movL);
16525 // peepconstraint (0.dst == 1.dst);
16526 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16527 // %}
16528
16529 // peephole
16530 // %{
16531 // peepmatch (addP_iReg_imm movP);
16532 // peepconstraint (0.dst == 1.dst);
16533 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16534 // %}
16535
16536 // // Change load of spilled value to only a spill
16537 // instruct storeI(memory mem, iRegI src)
16538 // %{
16539 // match(Set mem (StoreI mem src));
16540 // %}
16541 //
16542 // instruct loadI(iRegINoSp dst, memory mem)
16543 // %{
16544 // match(Set dst (LoadI mem));
16545 // %}
16546 //
16547
16548 //----------SMARTSPILL RULES---------------------------------------------------
16549 // These must follow all instruction definitions as they use the names
16550 // defined in the instructions definitions.
16551
16552 // Local Variables:
16553 // mode: c++
16554 // End: