1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 bool use_far_branch = MacroAssembler::target_needs_far_branch(SharedRuntime::deopt_blob()->unpack());
1202 // far: adrp, add, blr; near: bl
1203 uint target_branch_instructions = use_far_branch ? 3 : 1;
1204 // target branch + one branch instruction
1205 uint deopt_handler_instructions = target_branch_instructions + 1;
1206 return deopt_handler_instructions * NativeInstruction::instruction_size;
1207 }
1208 };
1209
1210 class Node::PD {
1211 public:
1212 enum NodeFlags {
1213 _last_flag = Node::_last_flag
1214 };
1215 };
1216
1217 bool is_CAS(int opcode, bool maybe_volatile);
1218
1219 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1220
1221 bool unnecessary_acquire(const Node *barrier);
1222 bool needs_acquiring_load(const Node *load);
1223
1224 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1225
1226 bool unnecessary_release(const Node *barrier);
1227 bool unnecessary_volatile(const Node *barrier);
1228 bool needs_releasing_store(const Node *store);
1229
1230 // predicate controlling translation of CompareAndSwapX
1231 bool needs_acquiring_load_exclusive(const Node *load);
1232
1233 // predicate controlling addressing modes
1234 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1235
1236 // Convert BoolTest condition to Assembler condition.
1237 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1238 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1239 %}
1240
1241 source %{
1242
1243 // Derived RegMask with conditionally allocatable registers
1244
1245 void PhaseOutput::pd_perform_mach_node_analysis() {
1246 }
1247
1248 int MachNode::pd_alignment_required() const {
1249 return 1;
1250 }
1251
1252 int MachNode::compute_padding(int current_offset) const {
1253 return 0;
1254 }
1255
1256 RegMask _ANY_REG32_mask;
1257 RegMask _ANY_REG_mask;
1258 RegMask _PTR_REG_mask;
1259 RegMask _NO_SPECIAL_REG32_mask;
1260 RegMask _NO_SPECIAL_REG_mask;
1261 RegMask _NO_SPECIAL_PTR_REG_mask;
1262 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1263
1264 void reg_mask_init() {
1265 // We derive below RegMask(s) from the ones which are auto-generated from
1266 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1267 // registers conditionally reserved.
1268
1269 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1270 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1271
1272 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1273
1274 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1275
1276 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1277 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1278
1279 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1280 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1281
1282 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1283 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1284
1285 // r27 is not allocatable when compressed oops is on and heapbase is not
1286 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1287 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1288 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1289 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1290 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1291 }
1292
1293 // r29 is not allocatable when PreserveFramePointer is on
1294 if (PreserveFramePointer) {
1295 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1296 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1297 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1301 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1302 }
1303
1304 // Optimizaton of volatile gets and puts
1305 // -------------------------------------
1306 //
1307 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1308 // use to implement volatile reads and writes. For a volatile read
1309 // we simply need
1310 //
1311 // ldar<x>
1312 //
1313 // and for a volatile write we need
1314 //
1315 // stlr<x>
1316 //
1317 // Alternatively, we can implement them by pairing a normal
1318 // load/store with a memory barrier. For a volatile read we need
1319 //
1320 // ldr<x>
1321 // dmb ishld
1322 //
1323 // for a volatile write
1324 //
1325 // dmb ish
1326 // str<x>
1327 // dmb ish
1328 //
1329 // We can also use ldaxr and stlxr to implement compare and swap CAS
1330 // sequences. These are normally translated to an instruction
1331 // sequence like the following
1332 //
1333 // dmb ish
1334 // retry:
1335 // ldxr<x> rval raddr
1336 // cmp rval rold
1337 // b.ne done
1338 // stlxr<x> rval, rnew, rold
1339 // cbnz rval retry
1340 // done:
1341 // cset r0, eq
1342 // dmb ishld
1343 //
1344 // Note that the exclusive store is already using an stlxr
1345 // instruction. That is required to ensure visibility to other
1346 // threads of the exclusive write (assuming it succeeds) before that
1347 // of any subsequent writes.
1348 //
1349 // The following instruction sequence is an improvement on the above
1350 //
1351 // retry:
1352 // ldaxr<x> rval raddr
1353 // cmp rval rold
1354 // b.ne done
1355 // stlxr<x> rval, rnew, rold
1356 // cbnz rval retry
1357 // done:
1358 // cset r0, eq
1359 //
1360 // We don't need the leading dmb ish since the stlxr guarantees
1361 // visibility of prior writes in the case that the swap is
1362 // successful. Crucially we don't have to worry about the case where
1363 // the swap is not successful since no valid program should be
1364 // relying on visibility of prior changes by the attempting thread
1365 // in the case where the CAS fails.
1366 //
1367 // Similarly, we don't need the trailing dmb ishld if we substitute
1368 // an ldaxr instruction since that will provide all the guarantees we
1369 // require regarding observation of changes made by other threads
1370 // before any change to the CAS address observed by the load.
1371 //
1372 // In order to generate the desired instruction sequence we need to
1373 // be able to identify specific 'signature' ideal graph node
1374 // sequences which i) occur as a translation of a volatile reads or
1375 // writes or CAS operations and ii) do not occur through any other
1376 // translation or graph transformation. We can then provide
1377 // alternative aldc matching rules which translate these node
1378 // sequences to the desired machine code sequences. Selection of the
1379 // alternative rules can be implemented by predicates which identify
1380 // the relevant node sequences.
1381 //
1382 // The ideal graph generator translates a volatile read to the node
1383 // sequence
1384 //
1385 // LoadX[mo_acquire]
1386 // MemBarAcquire
1387 //
1388 // As a special case when using the compressed oops optimization we
1389 // may also see this variant
1390 //
1391 // LoadN[mo_acquire]
1392 // DecodeN
1393 // MemBarAcquire
1394 //
1395 // A volatile write is translated to the node sequence
1396 //
1397 // MemBarRelease
1398 // StoreX[mo_release] {CardMark}-optional
1399 // MemBarVolatile
1400 //
1401 // n.b. the above node patterns are generated with a strict
1402 // 'signature' configuration of input and output dependencies (see
1403 // the predicates below for exact details). The card mark may be as
1404 // simple as a few extra nodes or, in a few GC configurations, may
1405 // include more complex control flow between the leading and
1406 // trailing memory barriers. However, whatever the card mark
1407 // configuration these signatures are unique to translated volatile
1408 // reads/stores -- they will not appear as a result of any other
1409 // bytecode translation or inlining nor as a consequence of
1410 // optimizing transforms.
1411 //
1412 // We also want to catch inlined unsafe volatile gets and puts and
1413 // be able to implement them using either ldar<x>/stlr<x> or some
1414 // combination of ldr<x>/stlr<x> and dmb instructions.
1415 //
1416 // Inlined unsafe volatiles puts manifest as a minor variant of the
1417 // normal volatile put node sequence containing an extra cpuorder
1418 // membar
1419 //
1420 // MemBarRelease
1421 // MemBarCPUOrder
1422 // StoreX[mo_release] {CardMark}-optional
1423 // MemBarCPUOrder
1424 // MemBarVolatile
1425 //
1426 // n.b. as an aside, a cpuorder membar is not itself subject to
1427 // matching and translation by adlc rules. However, the rule
1428 // predicates need to detect its presence in order to correctly
1429 // select the desired adlc rules.
1430 //
1431 // Inlined unsafe volatile gets manifest as a slightly different
1432 // node sequence to a normal volatile get because of the
1433 // introduction of some CPUOrder memory barriers to bracket the
1434 // Load. However, but the same basic skeleton of a LoadX feeding a
1435 // MemBarAcquire, possibly through an optional DecodeN, is still
1436 // present
1437 //
1438 // MemBarCPUOrder
1439 // || \\
1440 // MemBarCPUOrder LoadX[mo_acquire]
1441 // || |
1442 // || {DecodeN} optional
1443 // || /
1444 // MemBarAcquire
1445 //
1446 // In this case the acquire membar does not directly depend on the
1447 // load. However, we can be sure that the load is generated from an
1448 // inlined unsafe volatile get if we see it dependent on this unique
1449 // sequence of membar nodes. Similarly, given an acquire membar we
1450 // can know that it was added because of an inlined unsafe volatile
1451 // get if it is fed and feeds a cpuorder membar and if its feed
1452 // membar also feeds an acquiring load.
1453 //
1454 // Finally an inlined (Unsafe) CAS operation is translated to the
1455 // following ideal graph
1456 //
1457 // MemBarRelease
1458 // MemBarCPUOrder
1459 // CompareAndSwapX {CardMark}-optional
1460 // MemBarCPUOrder
1461 // MemBarAcquire
1462 //
1463 // So, where we can identify these volatile read and write
1464 // signatures we can choose to plant either of the above two code
1465 // sequences. For a volatile read we can simply plant a normal
1466 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1467 // also choose to inhibit translation of the MemBarAcquire and
1468 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1469 //
1470 // When we recognise a volatile store signature we can choose to
1471 // plant at a dmb ish as a translation for the MemBarRelease, a
1472 // normal str<x> and then a dmb ish for the MemBarVolatile.
1473 // Alternatively, we can inhibit translation of the MemBarRelease
1474 // and MemBarVolatile and instead plant a simple stlr<x>
1475 // instruction.
1476 //
1477 // when we recognise a CAS signature we can choose to plant a dmb
1478 // ish as a translation for the MemBarRelease, the conventional
1479 // macro-instruction sequence for the CompareAndSwap node (which
1480 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1481 // Alternatively, we can elide generation of the dmb instructions
1482 // and plant the alternative CompareAndSwap macro-instruction
1483 // sequence (which uses ldaxr<x>).
1484 //
1485 // Of course, the above only applies when we see these signature
1486 // configurations. We still want to plant dmb instructions in any
1487 // other cases where we may see a MemBarAcquire, MemBarRelease or
1488 // MemBarVolatile. For example, at the end of a constructor which
1489 // writes final/volatile fields we will see a MemBarRelease
1490 // instruction and this needs a 'dmb ish' lest we risk the
1491 // constructed object being visible without making the
1492 // final/volatile field writes visible.
1493 //
1494 // n.b. the translation rules below which rely on detection of the
1495 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1496 // If we see anything other than the signature configurations we
1497 // always just translate the loads and stores to ldr<x> and str<x>
1498 // and translate acquire, release and volatile membars to the
1499 // relevant dmb instructions.
1500 //
1501
1502 // is_CAS(int opcode, bool maybe_volatile)
1503 //
1504 // return true if opcode is one of the possible CompareAndSwapX
1505 // values otherwise false.
1506
1507 bool is_CAS(int opcode, bool maybe_volatile)
1508 {
1509 switch(opcode) {
1510 // We handle these
1511 case Op_CompareAndSwapI:
1512 case Op_CompareAndSwapL:
1513 case Op_CompareAndSwapP:
1514 case Op_CompareAndSwapN:
1515 case Op_CompareAndSwapB:
1516 case Op_CompareAndSwapS:
1517 case Op_GetAndSetI:
1518 case Op_GetAndSetL:
1519 case Op_GetAndSetP:
1520 case Op_GetAndSetN:
1521 case Op_GetAndAddI:
1522 case Op_GetAndAddL:
1523 return true;
1524 case Op_CompareAndExchangeI:
1525 case Op_CompareAndExchangeN:
1526 case Op_CompareAndExchangeB:
1527 case Op_CompareAndExchangeS:
1528 case Op_CompareAndExchangeL:
1529 case Op_CompareAndExchangeP:
1530 case Op_WeakCompareAndSwapB:
1531 case Op_WeakCompareAndSwapS:
1532 case Op_WeakCompareAndSwapI:
1533 case Op_WeakCompareAndSwapL:
1534 case Op_WeakCompareAndSwapP:
1535 case Op_WeakCompareAndSwapN:
1536 return maybe_volatile;
1537 default:
1538 return false;
1539 }
1540 }
1541
1542 // helper to determine the maximum number of Phi nodes we may need to
1543 // traverse when searching from a card mark membar for the merge mem
1544 // feeding a trailing membar or vice versa
1545
1546 // predicates controlling emit of ldr<x>/ldar<x>
1547
1548 bool unnecessary_acquire(const Node *barrier)
1549 {
1550 assert(barrier->is_MemBar(), "expecting a membar");
1551
1552 MemBarNode* mb = barrier->as_MemBar();
1553
1554 if (mb->trailing_load()) {
1555 return true;
1556 }
1557
1558 if (mb->trailing_load_store()) {
1559 Node* load_store = mb->in(MemBarNode::Precedent);
1560 assert(load_store->is_LoadStore(), "unexpected graph shape");
1561 return is_CAS(load_store->Opcode(), true);
1562 }
1563
1564 return false;
1565 }
1566
1567 bool needs_acquiring_load(const Node *n)
1568 {
1569 assert(n->is_Load(), "expecting a load");
1570 LoadNode *ld = n->as_Load();
1571 return ld->is_acquire();
1572 }
1573
1574 bool unnecessary_release(const Node *n)
1575 {
1576 assert((n->is_MemBar() &&
1577 n->Opcode() == Op_MemBarRelease),
1578 "expecting a release membar");
1579
1580 MemBarNode *barrier = n->as_MemBar();
1581 if (!barrier->leading()) {
1582 return false;
1583 } else {
1584 Node* trailing = barrier->trailing_membar();
1585 MemBarNode* trailing_mb = trailing->as_MemBar();
1586 assert(trailing_mb->trailing(), "Not a trailing membar?");
1587 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1588
1589 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1590 if (mem->is_Store()) {
1591 assert(mem->as_Store()->is_release(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1593 return true;
1594 } else {
1595 assert(mem->is_LoadStore(), "");
1596 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1597 return is_CAS(mem->Opcode(), true);
1598 }
1599 }
1600 return false;
1601 }
1602
1603 bool unnecessary_volatile(const Node *n)
1604 {
1605 // assert n->is_MemBar();
1606 MemBarNode *mbvol = n->as_MemBar();
1607
1608 bool release = mbvol->trailing_store();
1609 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1610 #ifdef ASSERT
1611 if (release) {
1612 Node* leading = mbvol->leading_membar();
1613 assert(leading->Opcode() == Op_MemBarRelease, "");
1614 assert(leading->as_MemBar()->leading_store(), "");
1615 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1616 }
1617 #endif
1618
1619 return release;
1620 }
1621
1622 // predicates controlling emit of str<x>/stlr<x>
1623
1624 bool needs_releasing_store(const Node *n)
1625 {
1626 // assert n->is_Store();
1627 StoreNode *st = n->as_Store();
1628 return st->trailing_membar() != nullptr;
1629 }
1630
1631 // predicate controlling translation of CAS
1632 //
1633 // returns true if CAS needs to use an acquiring load otherwise false
1634
1635 bool needs_acquiring_load_exclusive(const Node *n)
1636 {
1637 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1638 LoadStoreNode* ldst = n->as_LoadStore();
1639 if (is_CAS(n->Opcode(), false)) {
1640 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1641 } else {
1642 return ldst->trailing_membar() != nullptr;
1643 }
1644
1645 // so we can just return true here
1646 return true;
1647 }
1648
1649 #define __ masm->
1650
1651 // advance declarations for helper functions to convert register
1652 // indices to register objects
1653
1654 // the ad file has to provide implementations of certain methods
1655 // expected by the generic code
1656 //
1657 // REQUIRED FUNCTIONALITY
1658
1659 //=============================================================================
1660
1661 // !!!!! Special hack to get all types of calls to specify the byte offset
1662 // from the start of the call to the point where the return address
1663 // will point.
1664
1665 int MachCallStaticJavaNode::ret_addr_offset()
1666 {
1667 // call should be a simple bl
1668 int off = 4;
1669 return off;
1670 }
1671
1672 int MachCallDynamicJavaNode::ret_addr_offset()
1673 {
1674 return 16; // movz, movk, movk, bl
1675 }
1676
1677 int MachCallRuntimeNode::ret_addr_offset() {
1678 // for generated stubs the call will be
1679 // bl(addr)
1680 // or with far branches
1681 // bl(trampoline_stub)
1682 // for real runtime callouts it will be six instructions
1683 // see aarch64_enc_java_to_runtime
1684 // adr(rscratch2, retaddr)
1685 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1686 // lea(rscratch1, RuntimeAddress(addr)
1687 // blr(rscratch1)
1688 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1689 if (cb) {
1690 return 1 * NativeInstruction::instruction_size;
1691 } else {
1692 return 6 * NativeInstruction::instruction_size;
1693 }
1694 }
1695
1696 //=============================================================================
1697
1698 #ifndef PRODUCT
1699 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1700 st->print("BREAKPOINT");
1701 }
1702 #endif
1703
1704 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1705 __ brk(0);
1706 }
1707
1708 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1709 return MachNode::size(ra_);
1710 }
1711
1712 //=============================================================================
1713
1714 #ifndef PRODUCT
1715 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1716 st->print("nop \t# %d bytes pad for loops and calls", _count);
1717 }
1718 #endif
1719
1720 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1721 for (int i = 0; i < _count; i++) {
1722 __ nop();
1723 }
1724 }
1725
1726 uint MachNopNode::size(PhaseRegAlloc*) const {
1727 return _count * NativeInstruction::instruction_size;
1728 }
1729
1730 //=============================================================================
1731 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1732
1733 int ConstantTable::calculate_table_base_offset() const {
1734 return 0; // absolute addressing, no offset
1735 }
1736
1737 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1738 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1739 ShouldNotReachHere();
1740 }
1741
1742 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1743 // Empty encoding
1744 }
1745
1746 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1747 return 0;
1748 }
1749
1750 #ifndef PRODUCT
1751 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1752 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1753 }
1754 #endif
1755
1756 #ifndef PRODUCT
1757 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1758 Compile* C = ra_->C;
1759
1760 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1761
1762 if (C->output()->need_stack_bang(framesize))
1763 st->print("# stack bang size=%d\n\t", framesize);
1764
1765 if (VM_Version::use_rop_protection()) {
1766 st->print("ldr zr, [lr]\n\t");
1767 st->print("paciaz\n\t");
1768 }
1769 if (framesize < ((1 << 9) + 2 * wordSize)) {
1770 st->print("sub sp, sp, #%d\n\t", framesize);
1771 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1772 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1773 } else {
1774 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1775 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1776 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1777 st->print("sub sp, sp, rscratch1");
1778 }
1779 if (C->stub_function() == nullptr) {
1780 st->print("\n\t");
1781 st->print("ldr rscratch1, [guard]\n\t");
1782 st->print("dmb ishld\n\t");
1783 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1784 st->print("cmp rscratch1, rscratch2\n\t");
1785 st->print("b.eq skip");
1786 st->print("\n\t");
1787 st->print("blr #nmethod_entry_barrier_stub\n\t");
1788 st->print("b skip\n\t");
1789 st->print("guard: int\n\t");
1790 st->print("\n\t");
1791 st->print("skip:\n\t");
1792 }
1793 }
1794 #endif
1795
1796 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1797 Compile* C = ra_->C;
1798
1799 // n.b. frame size includes space for return pc and rfp
1800 const int framesize = C->output()->frame_size_in_bytes();
1801
1802 if (C->clinit_barrier_on_entry()) {
1803 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1804
1805 Label L_skip_barrier;
1806
1807 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1808 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1809 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1810 __ bind(L_skip_barrier);
1811 }
1812
1813 if (C->max_vector_size() > 0) {
1814 __ reinitialize_ptrue();
1815 }
1816
1817 int bangsize = C->output()->bang_size_in_bytes();
1818 if (C->output()->need_stack_bang(bangsize))
1819 __ generate_stack_overflow_check(bangsize);
1820
1821 __ build_frame(framesize);
1822
1823 if (C->stub_function() == nullptr) {
1824 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1825 // Dummy labels for just measuring the code size
1826 Label dummy_slow_path;
1827 Label dummy_continuation;
1828 Label dummy_guard;
1829 Label* slow_path = &dummy_slow_path;
1830 Label* continuation = &dummy_continuation;
1831 Label* guard = &dummy_guard;
1832 if (!Compile::current()->output()->in_scratch_emit_size()) {
1833 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1834 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1835 Compile::current()->output()->add_stub(stub);
1836 slow_path = &stub->entry();
1837 continuation = &stub->continuation();
1838 guard = &stub->guard();
1839 }
1840 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1841 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1842 }
1843
1844 if (VerifyStackAtCalls) {
1845 Unimplemented();
1846 }
1847
1848 C->output()->set_frame_complete(__ offset());
1849
1850 if (C->has_mach_constant_base_node()) {
1851 // NOTE: We set the table base offset here because users might be
1852 // emitted before MachConstantBaseNode.
1853 ConstantTable& constant_table = C->output()->constant_table();
1854 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1855 }
1856 }
1857
1858 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1859 {
1860 return MachNode::size(ra_); // too many variables; just compute it
1861 // the hard way
1862 }
1863
1864 int MachPrologNode::reloc() const
1865 {
1866 return 0;
1867 }
1868
1869 //=============================================================================
1870
1871 #ifndef PRODUCT
1872 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1873 Compile* C = ra_->C;
1874 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1875
1876 st->print("# pop frame %d\n\t",framesize);
1877
1878 if (framesize == 0) {
1879 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1880 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1881 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1882 st->print("add sp, sp, #%d\n\t", framesize);
1883 } else {
1884 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1885 st->print("add sp, sp, rscratch1\n\t");
1886 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1887 }
1888 if (VM_Version::use_rop_protection()) {
1889 st->print("autiaz\n\t");
1890 st->print("ldr zr, [lr]\n\t");
1891 }
1892
1893 if (do_polling() && C->is_method_compilation()) {
1894 st->print("# test polling word\n\t");
1895 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1896 st->print("cmp sp, rscratch1\n\t");
1897 st->print("bhi #slow_path");
1898 }
1899 }
1900 #endif
1901
1902 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1903 Compile* C = ra_->C;
1904 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1905
1906 __ remove_frame(framesize);
1907
1908 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1909 __ reserved_stack_check();
1910 }
1911
1912 if (do_polling() && C->is_method_compilation()) {
1913 Label dummy_label;
1914 Label* code_stub = &dummy_label;
1915 if (!C->output()->in_scratch_emit_size()) {
1916 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1917 C->output()->add_stub(stub);
1918 code_stub = &stub->entry();
1919 }
1920 __ relocate(relocInfo::poll_return_type);
1921 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1922 }
1923 }
1924
1925 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1926 // Variable size. Determine dynamically.
1927 return MachNode::size(ra_);
1928 }
1929
1930 int MachEpilogNode::reloc() const {
1931 // Return number of relocatable values contained in this instruction.
1932 return 1; // 1 for polling page.
1933 }
1934
1935 const Pipeline * MachEpilogNode::pipeline() const {
1936 return MachNode::pipeline_class();
1937 }
1938
1939 //=============================================================================
1940
1941 static enum RC rc_class(OptoReg::Name reg) {
1942
1943 if (reg == OptoReg::Bad) {
1944 return rc_bad;
1945 }
1946
1947 // we have 32 int registers * 2 halves
1948 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1949
1950 if (reg < slots_of_int_registers) {
1951 return rc_int;
1952 }
1953
1954 // we have 32 float register * 8 halves
1955 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1956 if (reg < slots_of_int_registers + slots_of_float_registers) {
1957 return rc_float;
1958 }
1959
1960 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1961 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1962 return rc_predicate;
1963 }
1964
1965 // Between predicate regs & stack is the flags.
1966 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1967
1968 return rc_stack;
1969 }
1970
1971 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1972 Compile* C = ra_->C;
1973
1974 // Get registers to move.
1975 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1976 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1977 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1978 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1979
1980 enum RC src_hi_rc = rc_class(src_hi);
1981 enum RC src_lo_rc = rc_class(src_lo);
1982 enum RC dst_hi_rc = rc_class(dst_hi);
1983 enum RC dst_lo_rc = rc_class(dst_lo);
1984
1985 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1986
1987 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1988 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1989 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1990 "expected aligned-adjacent pairs");
1991 }
1992
1993 if (src_lo == dst_lo && src_hi == dst_hi) {
1994 return 0; // Self copy, no move.
1995 }
1996
1997 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1998 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1999 int src_offset = ra_->reg2offset(src_lo);
2000 int dst_offset = ra_->reg2offset(dst_lo);
2001
2002 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2003 uint ireg = ideal_reg();
2004 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2005 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2006 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2007 if (ireg == Op_VecA && masm) {
2008 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2009 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2010 // stack->stack
2011 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2012 sve_vector_reg_size_in_bytes);
2013 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2014 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2015 sve_vector_reg_size_in_bytes);
2016 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2017 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2018 sve_vector_reg_size_in_bytes);
2019 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2020 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2021 as_FloatRegister(Matcher::_regEncode[src_lo]),
2022 as_FloatRegister(Matcher::_regEncode[src_lo]));
2023 } else {
2024 ShouldNotReachHere();
2025 }
2026 } else if (masm) {
2027 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2028 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2029 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2030 // stack->stack
2031 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2032 if (ireg == Op_VecD) {
2033 __ unspill(rscratch1, true, src_offset);
2034 __ spill(rscratch1, true, dst_offset);
2035 } else {
2036 __ spill_copy128(src_offset, dst_offset);
2037 }
2038 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2039 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 ireg == Op_VecD ? __ T8B : __ T16B,
2041 as_FloatRegister(Matcher::_regEncode[src_lo]));
2042 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2043 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2044 ireg == Op_VecD ? __ D : __ Q,
2045 ra_->reg2offset(dst_lo));
2046 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2047 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2048 ireg == Op_VecD ? __ D : __ Q,
2049 ra_->reg2offset(src_lo));
2050 } else {
2051 ShouldNotReachHere();
2052 }
2053 }
2054 } else if (masm) {
2055 switch (src_lo_rc) {
2056 case rc_int:
2057 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2058 if (is64) {
2059 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2060 as_Register(Matcher::_regEncode[src_lo]));
2061 } else {
2062 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2063 as_Register(Matcher::_regEncode[src_lo]));
2064 }
2065 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2066 if (is64) {
2067 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2068 as_Register(Matcher::_regEncode[src_lo]));
2069 } else {
2070 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2071 as_Register(Matcher::_regEncode[src_lo]));
2072 }
2073 } else { // gpr --> stack spill
2074 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2075 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2076 }
2077 break;
2078 case rc_float:
2079 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2080 if (is64) {
2081 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2082 as_FloatRegister(Matcher::_regEncode[src_lo]));
2083 } else {
2084 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2085 as_FloatRegister(Matcher::_regEncode[src_lo]));
2086 }
2087 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2088 if (is64) {
2089 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2090 as_FloatRegister(Matcher::_regEncode[src_lo]));
2091 } else {
2092 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2093 as_FloatRegister(Matcher::_regEncode[src_lo]));
2094 }
2095 } else { // fpr --> stack spill
2096 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2097 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2098 is64 ? __ D : __ S, dst_offset);
2099 }
2100 break;
2101 case rc_stack:
2102 if (dst_lo_rc == rc_int) { // stack --> gpr load
2103 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2104 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2105 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2106 is64 ? __ D : __ S, src_offset);
2107 } else if (dst_lo_rc == rc_predicate) {
2108 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2109 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2110 } else { // stack --> stack copy
2111 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2112 if (ideal_reg() == Op_RegVectMask) {
2113 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2114 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2115 } else {
2116 __ unspill(rscratch1, is64, src_offset);
2117 __ spill(rscratch1, is64, dst_offset);
2118 }
2119 }
2120 break;
2121 case rc_predicate:
2122 if (dst_lo_rc == rc_predicate) {
2123 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2124 } else if (dst_lo_rc == rc_stack) {
2125 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2126 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2127 } else {
2128 assert(false, "bad src and dst rc_class combination.");
2129 ShouldNotReachHere();
2130 }
2131 break;
2132 default:
2133 assert(false, "bad rc_class for spill");
2134 ShouldNotReachHere();
2135 }
2136 }
2137
2138 if (st) {
2139 st->print("spill ");
2140 if (src_lo_rc == rc_stack) {
2141 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2142 } else {
2143 st->print("%s -> ", Matcher::regName[src_lo]);
2144 }
2145 if (dst_lo_rc == rc_stack) {
2146 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2147 } else {
2148 st->print("%s", Matcher::regName[dst_lo]);
2149 }
2150 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2151 int vsize = 0;
2152 switch (ideal_reg()) {
2153 case Op_VecD:
2154 vsize = 64;
2155 break;
2156 case Op_VecX:
2157 vsize = 128;
2158 break;
2159 case Op_VecA:
2160 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2161 break;
2162 default:
2163 assert(false, "bad register type for spill");
2164 ShouldNotReachHere();
2165 }
2166 st->print("\t# vector spill size = %d", vsize);
2167 } else if (ideal_reg() == Op_RegVectMask) {
2168 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2169 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2170 st->print("\t# predicate spill size = %d", vsize);
2171 } else {
2172 st->print("\t# spill size = %d", is64 ? 64 : 32);
2173 }
2174 }
2175
2176 return 0;
2177
2178 }
2179
2180 #ifndef PRODUCT
2181 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2182 if (!ra_)
2183 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2184 else
2185 implementation(nullptr, ra_, false, st);
2186 }
2187 #endif
2188
2189 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2190 implementation(masm, ra_, false, nullptr);
2191 }
2192
2193 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2194 return MachNode::size(ra_);
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2201 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2202 int reg = ra_->get_reg_first(this);
2203 st->print("add %s, rsp, #%d]\t# box lock",
2204 Matcher::regName[reg], offset);
2205 }
2206 #endif
2207
2208 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2209 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2210 int reg = ra_->get_encode(this);
2211
2212 // This add will handle any 24-bit signed offset. 24 bits allows an
2213 // 8 megabyte stack frame.
2214 __ add(as_Register(reg), sp, offset);
2215 }
2216
2217 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2218 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2219 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2220
2221 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2222 return NativeInstruction::instruction_size;
2223 } else {
2224 return 2 * NativeInstruction::instruction_size;
2225 }
2226 }
2227
2228 //=============================================================================
2229
2230 #ifndef PRODUCT
2231 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2232 {
2233 st->print_cr("# MachUEPNode");
2234 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2235 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2236 st->print_cr("\tcmpw rscratch1, r10");
2237 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2238 }
2239 #endif
2240
2241 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2242 {
2243 __ ic_check(InteriorEntryAlignment);
2244 }
2245
2246 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2247 {
2248 return MachNode::size(ra_);
2249 }
2250
2251 // REQUIRED EMIT CODE
2252
2253 //=============================================================================
2254
2255 // Emit deopt handler code.
2256 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2257 {
2258 // Note that the code buffer's insts_mark is always relative to insts.
2259 // That's why we must use the macroassembler to generate a handler.
2260 address base = __ start_a_stub(size_deopt_handler());
2261 if (base == nullptr) {
2262 ciEnv::current()->record_failure("CodeCache is full");
2263 return 0; // CodeBuffer::expand failed
2264 }
2265
2266 int offset = __ offset();
2267 Label start;
2268 __ bind(start);
2269 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2270
2271 int entry_offset = __ offset();
2272 __ b(start);
2273
2274 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2275 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2276 "out of bounds read in post-call NOP check");
2277 __ end_a_stub();
2278 return entry_offset;
2279 }
2280
2281 // REQUIRED MATCHER CODE
2282
2283 //=============================================================================
2284
2285 bool Matcher::match_rule_supported(int opcode) {
2286 if (!has_match_rule(opcode))
2287 return false;
2288
2289 switch (opcode) {
2290 case Op_OnSpinWait:
2291 return VM_Version::supports_on_spin_wait();
2292 case Op_CacheWB:
2293 case Op_CacheWBPreSync:
2294 case Op_CacheWBPostSync:
2295 if (!VM_Version::supports_data_cache_line_flush()) {
2296 return false;
2297 }
2298 break;
2299 case Op_ExpandBits:
2300 case Op_CompressBits:
2301 if (!VM_Version::supports_svebitperm()) {
2302 return false;
2303 }
2304 break;
2305 case Op_FmaF:
2306 case Op_FmaD:
2307 case Op_FmaVF:
2308 case Op_FmaVD:
2309 if (!UseFMA) {
2310 return false;
2311 }
2312 break;
2313 case Op_FmaHF:
2314 // UseFMA flag also needs to be checked along with FEAT_FP16
2315 if (!UseFMA || !is_feat_fp16_supported()) {
2316 return false;
2317 }
2318 break;
2319 case Op_AddHF:
2320 case Op_SubHF:
2321 case Op_MulHF:
2322 case Op_DivHF:
2323 case Op_MinHF:
2324 case Op_MaxHF:
2325 case Op_SqrtHF:
2326 // Half-precision floating point scalar operations require FEAT_FP16
2327 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2328 // features are supported.
2329 if (!is_feat_fp16_supported()) {
2330 return false;
2331 }
2332 break;
2333 }
2334
2335 return true; // Per default match rules are supported.
2336 }
2337
2338 const RegMask* Matcher::predicate_reg_mask(void) {
2339 return &_PR_REG_mask;
2340 }
2341
2342 bool Matcher::supports_vector_calling_convention(void) {
2343 return EnableVectorSupport;
2344 }
2345
2346 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2347 assert(EnableVectorSupport, "sanity");
2348 int lo = V0_num;
2349 int hi = V0_H_num;
2350 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2351 hi = V0_K_num;
2352 }
2353 return OptoRegPair(hi, lo);
2354 }
2355
2356 // Is this branch offset short enough that a short branch can be used?
2357 //
2358 // NOTE: If the platform does not provide any short branch variants, then
2359 // this method should return false for offset 0.
2360 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2361 // The passed offset is relative to address of the branch.
2362
2363 return (-32768 <= offset && offset < 32768);
2364 }
2365
2366 // Vector width in bytes.
2367 int Matcher::vector_width_in_bytes(BasicType bt) {
2368 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2369 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2370 // Minimum 2 values in vector
2371 if (size < 2*type2aelembytes(bt)) size = 0;
2372 // But never < 4
2373 if (size < 4) size = 0;
2374 return size;
2375 }
2376
2377 // Limits on vector size (number of elements) loaded into vector.
2378 int Matcher::max_vector_size(const BasicType bt) {
2379 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2380 }
2381
2382 int Matcher::min_vector_size(const BasicType bt) {
2383 // Usually, the shortest vector length supported by AArch64 ISA and
2384 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2385 // vectors in a few special cases.
2386 int size;
2387 switch(bt) {
2388 case T_BOOLEAN:
2389 // Load/store a vector mask with only 2 elements for vector types
2390 // such as "2I/2F/2L/2D".
2391 size = 2;
2392 break;
2393 case T_BYTE:
2394 // Generate a "4B" vector, to support vector cast between "8B/16B"
2395 // and "4S/4I/4L/4F/4D".
2396 size = 4;
2397 break;
2398 case T_SHORT:
2399 // Generate a "2S" vector, to support vector cast between "4S/8S"
2400 // and "2I/2L/2F/2D".
2401 size = 2;
2402 break;
2403 default:
2404 // Limit the min vector length to 64-bit.
2405 size = 8 / type2aelembytes(bt);
2406 // The number of elements in a vector should be at least 2.
2407 size = MAX2(size, 2);
2408 }
2409
2410 int max_size = max_vector_size(bt);
2411 return MIN2(size, max_size);
2412 }
2413
2414 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2415 return Matcher::max_vector_size(bt);
2416 }
2417
2418 // Actual max scalable vector register length.
2419 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2420 return Matcher::max_vector_size(bt);
2421 }
2422
2423 // Vector ideal reg.
2424 uint Matcher::vector_ideal_reg(int len) {
2425 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2426 return Op_VecA;
2427 }
2428 switch(len) {
2429 // For 16-bit/32-bit mask vector, reuse VecD.
2430 case 2:
2431 case 4:
2432 case 8: return Op_VecD;
2433 case 16: return Op_VecX;
2434 }
2435 ShouldNotReachHere();
2436 return 0;
2437 }
2438
2439 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2440 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2441 switch (ideal_reg) {
2442 case Op_VecA: return new vecAOper();
2443 case Op_VecD: return new vecDOper();
2444 case Op_VecX: return new vecXOper();
2445 }
2446 ShouldNotReachHere();
2447 return nullptr;
2448 }
2449
2450 bool Matcher::is_reg2reg_move(MachNode* m) {
2451 return false;
2452 }
2453
2454 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2455 return false;
2456 }
2457
2458 bool Matcher::is_generic_vector(MachOper* opnd) {
2459 return opnd->opcode() == VREG;
2460 }
2461
2462 #ifdef ASSERT
2463 // Return whether or not this register is ever used as an argument.
2464 bool Matcher::can_be_java_arg(int reg)
2465 {
2466 return
2467 reg == R0_num || reg == R0_H_num ||
2468 reg == R1_num || reg == R1_H_num ||
2469 reg == R2_num || reg == R2_H_num ||
2470 reg == R3_num || reg == R3_H_num ||
2471 reg == R4_num || reg == R4_H_num ||
2472 reg == R5_num || reg == R5_H_num ||
2473 reg == R6_num || reg == R6_H_num ||
2474 reg == R7_num || reg == R7_H_num ||
2475 reg == V0_num || reg == V0_H_num ||
2476 reg == V1_num || reg == V1_H_num ||
2477 reg == V2_num || reg == V2_H_num ||
2478 reg == V3_num || reg == V3_H_num ||
2479 reg == V4_num || reg == V4_H_num ||
2480 reg == V5_num || reg == V5_H_num ||
2481 reg == V6_num || reg == V6_H_num ||
2482 reg == V7_num || reg == V7_H_num;
2483 }
2484 #endif
2485
2486 uint Matcher::int_pressure_limit()
2487 {
2488 // JDK-8183543: When taking the number of available registers as int
2489 // register pressure threshold, the jtreg test:
2490 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2491 // failed due to C2 compilation failure with
2492 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2493 //
2494 // A derived pointer is live at CallNode and then is flagged by RA
2495 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2496 // derived pointers and lastly fail to spill after reaching maximum
2497 // number of iterations. Lowering the default pressure threshold to
2498 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2499 // a high register pressure area of the code so that split_DEF can
2500 // generate DefinitionSpillCopy for the derived pointer.
2501 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2502 if (!PreserveFramePointer) {
2503 // When PreserveFramePointer is off, frame pointer is allocatable,
2504 // but different from other SOC registers, it is excluded from
2505 // fatproj's mask because its save type is No-Save. Decrease 1 to
2506 // ensure high pressure at fatproj when PreserveFramePointer is off.
2507 // See check_pressure_at_fatproj().
2508 default_int_pressure_threshold--;
2509 }
2510 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2511 }
2512
2513 uint Matcher::float_pressure_limit()
2514 {
2515 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2516 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2517 }
2518
2519 const RegMask& Matcher::firstI_proj_mask() {
2520 ShouldNotReachHere();
2521 return RegMask::EMPTY;
2522 }
2523
2524 // Register for the second projection of an int pair
2525 const RegMask& Matcher::secondI_proj_mask() {
2526 ShouldNotReachHere();
2527 return RegMask::EMPTY;
2528 }
2529
2530 // Register for the first projection of a long pair
2531 const RegMask& Matcher::firstL_proj_mask() {
2532 ShouldNotReachHere();
2533 return RegMask::EMPTY;
2534 }
2535
2536 // Register for the second projection of a long pair
2537 const RegMask& Matcher::secondL_proj_mask() {
2538 ShouldNotReachHere();
2539 return RegMask::EMPTY;
2540 }
2541
2542 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2543 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2544 Node* u = addp->fast_out(i);
2545 if (u->is_LoadStore()) {
2546 // On AArch64, LoadStoreNodes (i.e. compare and swap
2547 // instructions) only take register indirect as an operand, so
2548 // any attempt to use an AddPNode as an input to a LoadStoreNode
2549 // must fail.
2550 return false;
2551 }
2552 if (u->is_Mem()) {
2553 int opsize = u->as_Mem()->memory_size();
2554 assert(opsize > 0, "unexpected memory operand size");
2555 if (u->as_Mem()->memory_size() != (1<<shift)) {
2556 return false;
2557 }
2558 }
2559 }
2560 return true;
2561 }
2562
2563 // Convert BoolTest condition to Assembler condition.
2564 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2565 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2566 Assembler::Condition result;
2567 switch(cond) {
2568 case BoolTest::eq:
2569 result = Assembler::EQ; break;
2570 case BoolTest::ne:
2571 result = Assembler::NE; break;
2572 case BoolTest::le:
2573 result = Assembler::LE; break;
2574 case BoolTest::ge:
2575 result = Assembler::GE; break;
2576 case BoolTest::lt:
2577 result = Assembler::LT; break;
2578 case BoolTest::gt:
2579 result = Assembler::GT; break;
2580 case BoolTest::ule:
2581 result = Assembler::LS; break;
2582 case BoolTest::uge:
2583 result = Assembler::HS; break;
2584 case BoolTest::ult:
2585 result = Assembler::LO; break;
2586 case BoolTest::ugt:
2587 result = Assembler::HI; break;
2588 case BoolTest::overflow:
2589 result = Assembler::VS; break;
2590 case BoolTest::no_overflow:
2591 result = Assembler::VC; break;
2592 default:
2593 ShouldNotReachHere();
2594 return Assembler::Condition(-1);
2595 }
2596
2597 // Check conversion
2598 if (cond & BoolTest::unsigned_compare) {
2599 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2600 } else {
2601 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2602 }
2603
2604 return result;
2605 }
2606
2607 // Binary src (Replicate con)
2608 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2609 if (n == nullptr || m == nullptr) {
2610 return false;
2611 }
2612
2613 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2614 return false;
2615 }
2616
2617 Node* imm_node = m->in(1);
2618 if (!imm_node->is_Con()) {
2619 return false;
2620 }
2621
2622 const Type* t = imm_node->bottom_type();
2623 if (!(t->isa_int() || t->isa_long())) {
2624 return false;
2625 }
2626
2627 switch (n->Opcode()) {
2628 case Op_AndV:
2629 case Op_OrV:
2630 case Op_XorV: {
2631 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2632 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2633 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2634 }
2635 case Op_AddVB:
2636 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2637 case Op_AddVS:
2638 case Op_AddVI:
2639 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2640 case Op_AddVL:
2641 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2642 default:
2643 return false;
2644 }
2645 }
2646
2647 // (XorV src (Replicate m1))
2648 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2649 if (n != nullptr && m != nullptr) {
2650 return n->Opcode() == Op_XorV &&
2651 VectorNode::is_all_ones_vector(m);
2652 }
2653 return false;
2654 }
2655
2656 // Returns true if (n, m) matches "(XorVMask vm2 (MaskAll m1))" and that XorVMask
2657 // is used only by an AndVMask. In that case, cloning m (the MaskAll) lets the
2658 // matcher avoid sharing the MaskAll node and subsume the pattern into rule:
2659 // "(AndVMask vm1 (XorVMask vm2 (MaskAll m1)))".
2660 //
2661 // Limitation: the "andNot" rule still cannot be matched if "m" has other
2662 // uses outside this pattern.
2663 static bool is_vector_mask_not_operand_in_andnot_pattern(Node* n, Node* m) {
2664 if (n == nullptr || m == nullptr) {
2665 return false;
2666 }
2667
2668 if (VectorNode::is_all_ones_vector(m) &&
2669 n->Opcode() == Op_XorVMask &&
2670 n->outcnt() == 1 &&
2671 n->unique_out()->Opcode() == Op_AndVMask) {
2672 // If another input of the AndVMask is also a mask-not pattern that would
2673 // qualify for the `maskAll` cloning, do not clone the "maskAll" here,
2674 // because the match rule can only consume one such pattern.
2675 Node* use = n->unique_out();
2676 Node* other_input = use->in(1) == n ? use->in(2) : use->in(1);
2677 return !VectorNode::is_vectormask_bitwise_not_pattern(other_input);
2678 }
2679 return false;
2680 }
2681
2682 // Should the matcher clone input 'm' of node 'n'?
2683 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2684 if (is_vshift_con_pattern(n, m) ||
2685 is_vector_bitwise_not_pattern(n, m) ||
2686 is_vector_mask_not_operand_in_andnot_pattern(n, m) ||
2687 is_valid_sve_arith_imm_pattern(n, m) ||
2688 is_encode_and_store_pattern(n, m)) {
2689 mstack.push(m, Visit);
2690 return true;
2691 }
2692 return false;
2693 }
2694
2695 // Should the Matcher clone shifts on addressing modes, expecting them
2696 // to be subsumed into complex addressing expressions or compute them
2697 // into registers?
2698 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2699
2700 // Loads and stores with indirect memory input (e.g., volatile loads and
2701 // stores) do not subsume the input into complex addressing expressions. If
2702 // the addressing expression is input to at least one such load or store, do
2703 // not clone the addressing expression. Query needs_acquiring_load and
2704 // needs_releasing_store as a proxy for indirect memory input, as it is not
2705 // possible to directly query for indirect memory input at this stage.
2706 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2707 Node* n = m->fast_out(i);
2708 if (n->is_Load() && needs_acquiring_load(n)) {
2709 return false;
2710 }
2711 if (n->is_Store() && needs_releasing_store(n)) {
2712 return false;
2713 }
2714 }
2715
2716 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2717 return true;
2718 }
2719
2720 Node *off = m->in(AddPNode::Offset);
2721 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2722 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2723 // Are there other uses besides address expressions?
2724 !is_visited(off)) {
2725 address_visited.set(off->_idx); // Flag as address_visited
2726 mstack.push(off->in(2), Visit);
2727 Node *conv = off->in(1);
2728 if (conv->Opcode() == Op_ConvI2L &&
2729 // Are there other uses besides address expressions?
2730 !is_visited(conv)) {
2731 address_visited.set(conv->_idx); // Flag as address_visited
2732 mstack.push(conv->in(1), Pre_Visit);
2733 } else {
2734 mstack.push(conv, Pre_Visit);
2735 }
2736 address_visited.test_set(m->_idx); // Flag as address_visited
2737 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2738 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2739 return true;
2740 } else if (off->Opcode() == Op_ConvI2L &&
2741 // Are there other uses besides address expressions?
2742 !is_visited(off)) {
2743 address_visited.test_set(m->_idx); // Flag as address_visited
2744 address_visited.set(off->_idx); // Flag as address_visited
2745 mstack.push(off->in(1), Pre_Visit);
2746 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2747 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2748 return true;
2749 }
2750 return false;
2751 }
2752
2753 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2754 { \
2755 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2756 guarantee(DISP == 0, "mode not permitted for volatile"); \
2757 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2758 __ INSN(REG, as_Register(BASE)); \
2759 }
2760
2761
2762 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2763 {
2764 Address::extend scale;
2765
2766 // Hooboy, this is fugly. We need a way to communicate to the
2767 // encoder that the index needs to be sign extended, so we have to
2768 // enumerate all the cases.
2769 switch (opcode) {
2770 case INDINDEXSCALEDI2L:
2771 case INDINDEXSCALEDI2LN:
2772 case INDINDEXI2L:
2773 case INDINDEXI2LN:
2774 scale = Address::sxtw(size);
2775 break;
2776 default:
2777 scale = Address::lsl(size);
2778 }
2779
2780 if (index == -1) {
2781 return Address(base, disp);
2782 } else {
2783 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2784 return Address(base, as_Register(index), scale);
2785 }
2786 }
2787
2788
2789 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2790 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2791 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2792 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2793 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2794
2795 // Used for all non-volatile memory accesses. The use of
2796 // $mem->opcode() to discover whether this pattern uses sign-extended
2797 // offsets is something of a kludge.
2798 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2799 Register reg, int opcode,
2800 Register base, int index, int scale, int disp,
2801 int size_in_memory)
2802 {
2803 Address addr = mem2address(opcode, base, index, scale, disp);
2804 if (addr.getMode() == Address::base_plus_offset) {
2805 /* Fix up any out-of-range offsets. */
2806 assert_different_registers(rscratch1, base);
2807 assert_different_registers(rscratch1, reg);
2808 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2809 }
2810 (masm->*insn)(reg, addr);
2811 }
2812
2813 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2814 FloatRegister reg, int opcode,
2815 Register base, int index, int size, int disp,
2816 int size_in_memory)
2817 {
2818 Address::extend scale;
2819
2820 switch (opcode) {
2821 case INDINDEXSCALEDI2L:
2822 case INDINDEXSCALEDI2LN:
2823 scale = Address::sxtw(size);
2824 break;
2825 default:
2826 scale = Address::lsl(size);
2827 }
2828
2829 if (index == -1) {
2830 // Fix up any out-of-range offsets.
2831 assert_different_registers(rscratch1, base);
2832 Address addr = Address(base, disp);
2833 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2834 (masm->*insn)(reg, addr);
2835 } else {
2836 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2837 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2838 }
2839 }
2840
2841 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2842 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2843 int opcode, Register base, int index, int size, int disp)
2844 {
2845 if (index == -1) {
2846 (masm->*insn)(reg, T, Address(base, disp));
2847 } else {
2848 assert(disp == 0, "unsupported address mode");
2849 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2850 }
2851 }
2852
2853 %}
2854
2855
2856
2857 //----------ENCODING BLOCK-----------------------------------------------------
2858 // This block specifies the encoding classes used by the compiler to
2859 // output byte streams. Encoding classes are parameterized macros
2860 // used by Machine Instruction Nodes in order to generate the bit
2861 // encoding of the instruction. Operands specify their base encoding
2862 // interface with the interface keyword. There are currently
2863 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2864 // COND_INTER. REG_INTER causes an operand to generate a function
2865 // which returns its register number when queried. CONST_INTER causes
2866 // an operand to generate a function which returns the value of the
2867 // constant when queried. MEMORY_INTER causes an operand to generate
2868 // four functions which return the Base Register, the Index Register,
2869 // the Scale Value, and the Offset Value of the operand when queried.
2870 // COND_INTER causes an operand to generate six functions which return
2871 // the encoding code (ie - encoding bits for the instruction)
2872 // associated with each basic boolean condition for a conditional
2873 // instruction.
2874 //
2875 // Instructions specify two basic values for encoding. Again, a
2876 // function is available to check if the constant displacement is an
2877 // oop. They use the ins_encode keyword to specify their encoding
2878 // classes (which must be a sequence of enc_class names, and their
2879 // parameters, specified in the encoding block), and they use the
2880 // opcode keyword to specify, in order, their primary, secondary, and
2881 // tertiary opcode. Only the opcode sections which a particular
2882 // instruction needs for encoding need to be specified.
2883 encode %{
2884 // Build emit functions for each basic byte or larger field in the
2885 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2886 // from C++ code in the enc_class source block. Emit functions will
2887 // live in the main source block for now. In future, we can
2888 // generalize this by adding a syntax that specifies the sizes of
2889 // fields in an order, so that the adlc can build the emit functions
2890 // automagically
2891
2892 // catch all for unimplemented encodings
2893 enc_class enc_unimplemented %{
2894 __ unimplemented("C2 catch all");
2895 %}
2896
2897 // BEGIN Non-volatile memory access
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2974 Register dst_reg = as_Register($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2982 Register dst_reg = as_Register($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2990 Register dst_reg = as_Register($dst$$reg);
2991 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2998 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2999 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3006 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3007 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3008 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3009 %}
3010
3011 // This encoding class is generated automatically from ad_encode.m4.
3012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3013 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3014 Register src_reg = as_Register($src$$reg);
3015 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strb0(memory1 mem) %{
3022 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3029 Register src_reg = as_Register($src$$reg);
3030 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_strh0(memory2 mem) %{
3037 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3044 Register src_reg = as_Register($src$$reg);
3045 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3046 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3047 %}
3048
3049 // This encoding class is generated automatically from ad_encode.m4.
3050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3051 enc_class aarch64_enc_strw0(memory4 mem) %{
3052 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3054 %}
3055
3056 // This encoding class is generated automatically from ad_encode.m4.
3057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3058 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3059 Register src_reg = as_Register($src$$reg);
3060 // we sometimes get asked to store the stack pointer into the
3061 // current thread -- we cannot do that directly on AArch64
3062 if (src_reg == r31_sp) {
3063 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3064 __ mov(rscratch2, sp);
3065 src_reg = rscratch2;
3066 }
3067 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3068 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3069 %}
3070
3071 // This encoding class is generated automatically from ad_encode.m4.
3072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3073 enc_class aarch64_enc_str0(memory8 mem) %{
3074 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3076 %}
3077
3078 // This encoding class is generated automatically from ad_encode.m4.
3079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3080 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3081 FloatRegister src_reg = as_FloatRegister($src$$reg);
3082 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3083 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3084 %}
3085
3086 // This encoding class is generated automatically from ad_encode.m4.
3087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3088 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3089 FloatRegister src_reg = as_FloatRegister($src$$reg);
3090 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3091 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3092 %}
3093
3094 // This encoding class is generated automatically from ad_encode.m4.
3095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3096 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3097 __ membar(Assembler::StoreStore);
3098 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3099 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3100 %}
3101
3102 // END Non-volatile memory access
3103
3104 // Vector loads and stores
3105 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3106 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3107 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3112 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3113 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3118 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3119 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3124 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3125 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3130 FloatRegister src_reg = as_FloatRegister($src$$reg);
3131 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3132 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3133 %}
3134
3135 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3136 FloatRegister src_reg = as_FloatRegister($src$$reg);
3137 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3138 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3139 %}
3140
3141 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3142 FloatRegister src_reg = as_FloatRegister($src$$reg);
3143 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3144 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3145 %}
3146
3147 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3148 FloatRegister src_reg = as_FloatRegister($src$$reg);
3149 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3150 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3151 %}
3152
3153 // volatile loads and stores
3154
3155 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3156 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3157 rscratch1, stlrb);
3158 %}
3159
3160 enc_class aarch64_enc_stlrb0(memory mem) %{
3161 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3162 rscratch1, stlrb);
3163 %}
3164
3165 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3166 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3167 rscratch1, stlrh);
3168 %}
3169
3170 enc_class aarch64_enc_stlrh0(memory mem) %{
3171 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3172 rscratch1, stlrh);
3173 %}
3174
3175 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3176 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, stlrw);
3178 %}
3179
3180 enc_class aarch64_enc_stlrw0(memory mem) %{
3181 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, stlrw);
3183 %}
3184
3185 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3186 Register dst_reg = as_Register($dst$$reg);
3187 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarb);
3189 __ sxtbw(dst_reg, dst_reg);
3190 %}
3191
3192 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3193 Register dst_reg = as_Register($dst$$reg);
3194 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarb);
3196 __ sxtb(dst_reg, dst_reg);
3197 %}
3198
3199 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3200 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3201 rscratch1, ldarb);
3202 %}
3203
3204 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3205 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, ldarb);
3207 %}
3208
3209 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3210 Register dst_reg = as_Register($dst$$reg);
3211 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, ldarh);
3213 __ sxthw(dst_reg, dst_reg);
3214 %}
3215
3216 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3217 Register dst_reg = as_Register($dst$$reg);
3218 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, ldarh);
3220 __ sxth(dst_reg, dst_reg);
3221 %}
3222
3223 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3224 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3225 rscratch1, ldarh);
3226 %}
3227
3228 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3229 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3230 rscratch1, ldarh);
3231 %}
3232
3233 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3234 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3235 rscratch1, ldarw);
3236 %}
3237
3238 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3239 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3240 rscratch1, ldarw);
3241 %}
3242
3243 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3244 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3245 rscratch1, ldar);
3246 %}
3247
3248 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3249 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3250 rscratch1, ldarw);
3251 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3252 %}
3253
3254 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3255 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3256 rscratch1, ldar);
3257 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3258 %}
3259
3260 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3261 Register src_reg = as_Register($src$$reg);
3262 // we sometimes get asked to store the stack pointer into the
3263 // current thread -- we cannot do that directly on AArch64
3264 if (src_reg == r31_sp) {
3265 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3266 __ mov(rscratch2, sp);
3267 src_reg = rscratch2;
3268 }
3269 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3270 rscratch1, stlr);
3271 %}
3272
3273 enc_class aarch64_enc_stlr0(memory mem) %{
3274 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3275 rscratch1, stlr);
3276 %}
3277
3278 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3279 {
3280 FloatRegister src_reg = as_FloatRegister($src$$reg);
3281 __ fmovs(rscratch2, src_reg);
3282 }
3283 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3284 rscratch1, stlrw);
3285 %}
3286
3287 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3288 {
3289 FloatRegister src_reg = as_FloatRegister($src$$reg);
3290 __ fmovd(rscratch2, src_reg);
3291 }
3292 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3293 rscratch1, stlr);
3294 %}
3295
3296 // synchronized read/update encodings
3297
3298 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3299 Register dst_reg = as_Register($dst$$reg);
3300 Register base = as_Register($mem$$base);
3301 int index = $mem$$index;
3302 int scale = $mem$$scale;
3303 int disp = $mem$$disp;
3304 if (index == -1) {
3305 if (disp != 0) {
3306 __ lea(rscratch1, Address(base, disp));
3307 __ ldaxr(dst_reg, rscratch1);
3308 } else {
3309 // TODO
3310 // should we ever get anything other than this case?
3311 __ ldaxr(dst_reg, base);
3312 }
3313 } else {
3314 Register index_reg = as_Register(index);
3315 if (disp == 0) {
3316 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3317 __ ldaxr(dst_reg, rscratch1);
3318 } else {
3319 __ lea(rscratch1, Address(base, disp));
3320 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3321 __ ldaxr(dst_reg, rscratch1);
3322 }
3323 }
3324 %}
3325
3326 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3327 Register src_reg = as_Register($src$$reg);
3328 Register base = as_Register($mem$$base);
3329 int index = $mem$$index;
3330 int scale = $mem$$scale;
3331 int disp = $mem$$disp;
3332 if (index == -1) {
3333 if (disp != 0) {
3334 __ lea(rscratch2, Address(base, disp));
3335 __ stlxr(rscratch1, src_reg, rscratch2);
3336 } else {
3337 // TODO
3338 // should we ever get anything other than this case?
3339 __ stlxr(rscratch1, src_reg, base);
3340 }
3341 } else {
3342 Register index_reg = as_Register(index);
3343 if (disp == 0) {
3344 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3345 __ stlxr(rscratch1, src_reg, rscratch2);
3346 } else {
3347 __ lea(rscratch2, Address(base, disp));
3348 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3349 __ stlxr(rscratch1, src_reg, rscratch2);
3350 }
3351 }
3352 __ cmpw(rscratch1, zr);
3353 %}
3354
3355 // prefetch encodings
3356
3357 enc_class aarch64_enc_prefetchw(memory mem) %{
3358 Register base = as_Register($mem$$base);
3359 int index = $mem$$index;
3360 int scale = $mem$$scale;
3361 int disp = $mem$$disp;
3362 if (index == -1) {
3363 // Fix up any out-of-range offsets.
3364 assert_different_registers(rscratch1, base);
3365 Address addr = Address(base, disp);
3366 addr = __ legitimize_address(addr, 8, rscratch1);
3367 __ prfm(addr, PSTL1KEEP);
3368 } else {
3369 Register index_reg = as_Register(index);
3370 if (disp == 0) {
3371 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3372 } else {
3373 __ lea(rscratch1, Address(base, disp));
3374 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3375 }
3376 }
3377 %}
3378
3379 // mov encodings
3380
3381 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3382 uint32_t con = (uint32_t)$src$$constant;
3383 Register dst_reg = as_Register($dst$$reg);
3384 if (con == 0) {
3385 __ movw(dst_reg, zr);
3386 } else {
3387 __ movw(dst_reg, con);
3388 }
3389 %}
3390
3391 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3392 Register dst_reg = as_Register($dst$$reg);
3393 uint64_t con = (uint64_t)$src$$constant;
3394 if (con == 0) {
3395 __ mov(dst_reg, zr);
3396 } else {
3397 __ mov(dst_reg, con);
3398 }
3399 %}
3400
3401 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3402 Register dst_reg = as_Register($dst$$reg);
3403 address con = (address)$src$$constant;
3404 if (con == nullptr || con == (address)1) {
3405 ShouldNotReachHere();
3406 } else {
3407 relocInfo::relocType rtype = $src->constant_reloc();
3408 if (rtype == relocInfo::oop_type) {
3409 __ movoop(dst_reg, (jobject)con);
3410 } else if (rtype == relocInfo::metadata_type) {
3411 __ mov_metadata(dst_reg, (Metadata*)con);
3412 } else {
3413 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3414 // load fake address constants using a normal move
3415 if (! __ is_valid_AArch64_address(con) ||
3416 con < (address)(uintptr_t)os::vm_page_size() ||
3417 rtype == relocInfo::none) {
3418 __ mov(dst_reg, con);
3419 } else {
3420 // use shorter adrp/add sequence for external_word relocation
3421 uint64_t offset;
3422 __ adrp(dst_reg, Address(con, rtype), offset);
3423 __ add(dst_reg, dst_reg, offset);
3424 }
3425 }
3426 }
3427 %}
3428
3429 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3430 Register dst_reg = as_Register($dst$$reg);
3431 __ mov(dst_reg, zr);
3432 %}
3433
3434 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3435 Register dst_reg = as_Register($dst$$reg);
3436 __ mov(dst_reg, (uint64_t)1);
3437 %}
3438
3439 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 address con = (address)$src$$constant;
3442 if (con == nullptr) {
3443 ShouldNotReachHere();
3444 } else {
3445 relocInfo::relocType rtype = $src->constant_reloc();
3446 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3447 __ set_narrow_oop(dst_reg, (jobject)con);
3448 }
3449 %}
3450
3451 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3452 Register dst_reg = as_Register($dst$$reg);
3453 __ mov(dst_reg, zr);
3454 %}
3455
3456 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3457 Register dst_reg = as_Register($dst$$reg);
3458 address con = (address)$src$$constant;
3459 if (con == nullptr) {
3460 ShouldNotReachHere();
3461 } else {
3462 relocInfo::relocType rtype = $src->constant_reloc();
3463 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3464 __ set_narrow_klass(dst_reg, (Klass *)con);
3465 }
3466 %}
3467
3468 // arithmetic encodings
3469
3470 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3471 Register dst_reg = as_Register($dst$$reg);
3472 Register src_reg = as_Register($src1$$reg);
3473 int32_t con = (int32_t)$src2$$constant;
3474 // add has primary == 0, subtract has primary == 1
3475 if ($primary) { con = -con; }
3476 if (con < 0) {
3477 __ subw(dst_reg, src_reg, -con);
3478 } else {
3479 __ addw(dst_reg, src_reg, con);
3480 }
3481 %}
3482
3483 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3484 Register dst_reg = as_Register($dst$$reg);
3485 Register src_reg = as_Register($src1$$reg);
3486 int32_t con = (int32_t)$src2$$constant;
3487 // add has primary == 0, subtract has primary == 1
3488 if ($primary) { con = -con; }
3489 if (con < 0) {
3490 __ sub(dst_reg, src_reg, -con);
3491 } else {
3492 __ add(dst_reg, src_reg, con);
3493 }
3494 %}
3495
3496 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3497 Register dst_reg = as_Register($dst$$reg);
3498 Register src1_reg = as_Register($src1$$reg);
3499 Register src2_reg = as_Register($src2$$reg);
3500 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3501 %}
3502
3503 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3504 Register dst_reg = as_Register($dst$$reg);
3505 Register src1_reg = as_Register($src1$$reg);
3506 Register src2_reg = as_Register($src2$$reg);
3507 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3508 %}
3509
3510 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3511 Register dst_reg = as_Register($dst$$reg);
3512 Register src1_reg = as_Register($src1$$reg);
3513 Register src2_reg = as_Register($src2$$reg);
3514 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3515 %}
3516
3517 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3518 Register dst_reg = as_Register($dst$$reg);
3519 Register src1_reg = as_Register($src1$$reg);
3520 Register src2_reg = as_Register($src2$$reg);
3521 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3522 %}
3523
3524 // compare instruction encodings
3525
3526 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3527 Register reg1 = as_Register($src1$$reg);
3528 Register reg2 = as_Register($src2$$reg);
3529 __ cmpw(reg1, reg2);
3530 %}
3531
3532 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3533 Register reg = as_Register($src1$$reg);
3534 int32_t val = $src2$$constant;
3535 if (val >= 0) {
3536 __ subsw(zr, reg, val);
3537 } else {
3538 __ addsw(zr, reg, -val);
3539 }
3540 %}
3541
3542 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3543 Register reg1 = as_Register($src1$$reg);
3544 uint32_t val = (uint32_t)$src2$$constant;
3545 __ movw(rscratch1, val);
3546 __ cmpw(reg1, rscratch1);
3547 %}
3548
3549 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3550 Register reg1 = as_Register($src1$$reg);
3551 Register reg2 = as_Register($src2$$reg);
3552 __ cmp(reg1, reg2);
3553 %}
3554
3555 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3556 Register reg = as_Register($src1$$reg);
3557 int64_t val = $src2$$constant;
3558 if (val >= 0) {
3559 __ subs(zr, reg, val);
3560 } else if (val != -val) {
3561 __ adds(zr, reg, -val);
3562 } else {
3563 // aargh, Long.MIN_VALUE is a special case
3564 __ orr(rscratch1, zr, (uint64_t)val);
3565 __ subs(zr, reg, rscratch1);
3566 }
3567 %}
3568
3569 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3570 Register reg1 = as_Register($src1$$reg);
3571 uint64_t val = (uint64_t)$src2$$constant;
3572 __ mov(rscratch1, val);
3573 __ cmp(reg1, rscratch1);
3574 %}
3575
3576 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3577 Register reg1 = as_Register($src1$$reg);
3578 Register reg2 = as_Register($src2$$reg);
3579 __ cmp(reg1, reg2);
3580 %}
3581
3582 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3583 Register reg1 = as_Register($src1$$reg);
3584 Register reg2 = as_Register($src2$$reg);
3585 __ cmpw(reg1, reg2);
3586 %}
3587
3588 enc_class aarch64_enc_testp(iRegP src) %{
3589 Register reg = as_Register($src$$reg);
3590 __ cmp(reg, zr);
3591 %}
3592
3593 enc_class aarch64_enc_testn(iRegN src) %{
3594 Register reg = as_Register($src$$reg);
3595 __ cmpw(reg, zr);
3596 %}
3597
3598 enc_class aarch64_enc_b(label lbl) %{
3599 Label *L = $lbl$$label;
3600 __ b(*L);
3601 %}
3602
3603 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3604 Label *L = $lbl$$label;
3605 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3606 %}
3607
3608 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3609 Label *L = $lbl$$label;
3610 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3611 %}
3612
3613 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3614 %{
3615 Register sub_reg = as_Register($sub$$reg);
3616 Register super_reg = as_Register($super$$reg);
3617 Register temp_reg = as_Register($temp$$reg);
3618 Register result_reg = as_Register($result$$reg);
3619
3620 Label miss;
3621 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3622 nullptr, &miss,
3623 /*set_cond_codes:*/ true);
3624 if ($primary) {
3625 __ mov(result_reg, zr);
3626 }
3627 __ bind(miss);
3628 %}
3629
3630 enc_class aarch64_enc_java_static_call(method meth) %{
3631 address addr = (address)$meth$$method;
3632 address call;
3633 if (!_method) {
3634 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3635 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3636 if (call == nullptr) {
3637 ciEnv::current()->record_failure("CodeCache is full");
3638 return;
3639 }
3640 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3641 // The NOP here is purely to ensure that eliding a call to
3642 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3643 __ nop();
3644 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3645 } else {
3646 int method_index = resolved_method_index(masm);
3647 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3648 : static_call_Relocation::spec(method_index);
3649 call = __ trampoline_call(Address(addr, rspec));
3650 if (call == nullptr) {
3651 ciEnv::current()->record_failure("CodeCache is full");
3652 return;
3653 }
3654 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3655 // Calls of the same statically bound method can share
3656 // a stub to the interpreter.
3657 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3658 } else {
3659 // Emit stub for static call
3660 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3661 if (stub == nullptr) {
3662 ciEnv::current()->record_failure("CodeCache is full");
3663 return;
3664 }
3665 }
3666 }
3667
3668 __ post_call_nop();
3669
3670 // Only non uncommon_trap calls need to reinitialize ptrue.
3671 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3672 __ reinitialize_ptrue();
3673 }
3674 %}
3675
3676 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3677 int method_index = resolved_method_index(masm);
3678 address call = __ ic_call((address)$meth$$method, method_index);
3679 if (call == nullptr) {
3680 ciEnv::current()->record_failure("CodeCache is full");
3681 return;
3682 }
3683 __ post_call_nop();
3684 if (Compile::current()->max_vector_size() > 0) {
3685 __ reinitialize_ptrue();
3686 }
3687 %}
3688
3689 enc_class aarch64_enc_call_epilog() %{
3690 if (VerifyStackAtCalls) {
3691 // Check that stack depth is unchanged: find majik cookie on stack
3692 __ call_Unimplemented();
3693 }
3694 %}
3695
3696 enc_class aarch64_enc_java_to_runtime(method meth) %{
3697 // some calls to generated routines (arraycopy code) are scheduled
3698 // by C2 as runtime calls. if so we can call them using a br (they
3699 // will be in a reachable segment) otherwise we have to use a blr
3700 // which loads the absolute address into a register.
3701 address entry = (address)$meth$$method;
3702 CodeBlob *cb = CodeCache::find_blob(entry);
3703 if (cb) {
3704 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3705 if (call == nullptr) {
3706 ciEnv::current()->record_failure("CodeCache is full");
3707 return;
3708 }
3709 __ post_call_nop();
3710 } else {
3711 Label retaddr;
3712 // Make the anchor frame walkable
3713 __ adr(rscratch2, retaddr);
3714 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3715 __ lea(rscratch1, RuntimeAddress(entry));
3716 __ blr(rscratch1);
3717 __ bind(retaddr);
3718 __ post_call_nop();
3719 }
3720 if (Compile::current()->max_vector_size() > 0) {
3721 __ reinitialize_ptrue();
3722 }
3723 %}
3724
3725 enc_class aarch64_enc_rethrow() %{
3726 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3727 %}
3728
3729 enc_class aarch64_enc_ret() %{
3730 #ifdef ASSERT
3731 if (Compile::current()->max_vector_size() > 0) {
3732 __ verify_ptrue();
3733 }
3734 #endif
3735 __ ret(lr);
3736 %}
3737
3738 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3739 Register target_reg = as_Register($jump_target$$reg);
3740 __ br(target_reg);
3741 %}
3742
3743 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3744 Register target_reg = as_Register($jump_target$$reg);
3745 // exception oop should be in r0
3746 // ret addr has been popped into lr
3747 // callee expects it in r3
3748 __ mov(r3, lr);
3749 __ br(target_reg);
3750 %}
3751
3752 %}
3753
3754 //----------FRAME--------------------------------------------------------------
3755 // Definition of frame structure and management information.
3756 //
3757 // S T A C K L A Y O U T Allocators stack-slot number
3758 // | (to get allocators register number
3759 // G Owned by | | v add OptoReg::stack0())
3760 // r CALLER | |
3761 // o | +--------+ pad to even-align allocators stack-slot
3762 // w V | pad0 | numbers; owned by CALLER
3763 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3764 // h ^ | in | 5
3765 // | | args | 4 Holes in incoming args owned by SELF
3766 // | | | | 3
3767 // | | +--------+
3768 // V | | old out| Empty on Intel, window on Sparc
3769 // | old |preserve| Must be even aligned.
3770 // | SP-+--------+----> Matcher::_old_SP, even aligned
3771 // | | in | 3 area for Intel ret address
3772 // Owned by |preserve| Empty on Sparc.
3773 // SELF +--------+
3774 // | | pad2 | 2 pad to align old SP
3775 // | +--------+ 1
3776 // | | locks | 0
3777 // | +--------+----> OptoReg::stack0(), even aligned
3778 // | | pad1 | 11 pad to align new SP
3779 // | +--------+
3780 // | | | 10
3781 // | | spills | 9 spills
3782 // V | | 8 (pad0 slot for callee)
3783 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3784 // ^ | out | 7
3785 // | | args | 6 Holes in outgoing args owned by CALLEE
3786 // Owned by +--------+
3787 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3788 // | new |preserve| Must be even-aligned.
3789 // | SP-+--------+----> Matcher::_new_SP, even aligned
3790 // | | |
3791 //
3792 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3793 // known from SELF's arguments and the Java calling convention.
3794 // Region 6-7 is determined per call site.
3795 // Note 2: If the calling convention leaves holes in the incoming argument
3796 // area, those holes are owned by SELF. Holes in the outgoing area
3797 // are owned by the CALLEE. Holes should not be necessary in the
3798 // incoming area, as the Java calling convention is completely under
3799 // the control of the AD file. Doubles can be sorted and packed to
3800 // avoid holes. Holes in the outgoing arguments may be necessary for
3801 // varargs C calling conventions.
3802 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3803 // even aligned with pad0 as needed.
3804 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3805 // (the latter is true on Intel but is it false on AArch64?)
3806 // region 6-11 is even aligned; it may be padded out more so that
3807 // the region from SP to FP meets the minimum stack alignment.
3808 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3809 // alignment. Region 11, pad1, may be dynamically extended so that
3810 // SP meets the minimum alignment.
3811
3812 frame %{
3813 // These three registers define part of the calling convention
3814 // between compiled code and the interpreter.
3815
3816 // Inline Cache Register or Method for I2C.
3817 inline_cache_reg(R12);
3818
3819 // Number of stack slots consumed by locking an object
3820 sync_stack_slots(2);
3821
3822 // Compiled code's Frame Pointer
3823 frame_pointer(R31);
3824
3825 // Stack alignment requirement
3826 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3827
3828 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3829 // for calls to C. Supports the var-args backing area for register parms.
3830 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3831
3832 // The after-PROLOG location of the return address. Location of
3833 // return address specifies a type (REG or STACK) and a number
3834 // representing the register number (i.e. - use a register name) or
3835 // stack slot.
3836 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3837 // Otherwise, it is above the locks and verification slot and alignment word
3838 // TODO this may well be correct but need to check why that - 2 is there
3839 // ppc port uses 0 but we definitely need to allow for fixed_slots
3840 // which folds in the space used for monitors
3841 return_addr(STACK - 2 +
3842 align_up((Compile::current()->in_preserve_stack_slots() +
3843 Compile::current()->fixed_slots()),
3844 stack_alignment_in_slots()));
3845
3846 // Location of compiled Java return values. Same as C for now.
3847 return_value
3848 %{
3849 // TODO do we allow ideal_reg == Op_RegN???
3850 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3851 "only return normal values");
3852
3853 static const int lo[Op_RegL + 1] = { // enum name
3854 0, // Op_Node
3855 0, // Op_Set
3856 R0_num, // Op_RegN
3857 R0_num, // Op_RegI
3858 R0_num, // Op_RegP
3859 V0_num, // Op_RegF
3860 V0_num, // Op_RegD
3861 R0_num // Op_RegL
3862 };
3863
3864 static const int hi[Op_RegL + 1] = { // enum name
3865 0, // Op_Node
3866 0, // Op_Set
3867 OptoReg::Bad, // Op_RegN
3868 OptoReg::Bad, // Op_RegI
3869 R0_H_num, // Op_RegP
3870 OptoReg::Bad, // Op_RegF
3871 V0_H_num, // Op_RegD
3872 R0_H_num // Op_RegL
3873 };
3874
3875 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3876 %}
3877 %}
3878
3879 //----------ATTRIBUTES---------------------------------------------------------
3880 //----------Operand Attributes-------------------------------------------------
3881 op_attrib op_cost(1); // Required cost attribute
3882
3883 //----------Instruction Attributes---------------------------------------------
3884 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3885 ins_attrib ins_size(32); // Required size attribute (in bits)
3886 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3887 // a non-matching short branch variant
3888 // of some long branch?
3889 ins_attrib ins_alignment(4); // Required alignment attribute (must
3890 // be a power of 2) specifies the
3891 // alignment that some part of the
3892 // instruction (not necessarily the
3893 // start) requires. If > 1, a
3894 // compute_padding() function must be
3895 // provided for the instruction
3896
3897 // Whether this node is expanded during code emission into a sequence of
3898 // instructions and the first instruction can perform an implicit null check.
3899 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3900
3901 //----------OPERANDS-----------------------------------------------------------
3902 // Operand definitions must precede instruction definitions for correct parsing
3903 // in the ADLC because operands constitute user defined types which are used in
3904 // instruction definitions.
3905
3906 //----------Simple Operands----------------------------------------------------
3907
3908 // Integer operands 32 bit
3909 // 32 bit immediate
3910 operand immI()
3911 %{
3912 match(ConI);
3913
3914 op_cost(0);
3915 format %{ %}
3916 interface(CONST_INTER);
3917 %}
3918
3919 // 32 bit zero
3920 operand immI0()
3921 %{
3922 predicate(n->get_int() == 0);
3923 match(ConI);
3924
3925 op_cost(0);
3926 format %{ %}
3927 interface(CONST_INTER);
3928 %}
3929
3930 // 32 bit unit increment
3931 operand immI_1()
3932 %{
3933 predicate(n->get_int() == 1);
3934 match(ConI);
3935
3936 op_cost(0);
3937 format %{ %}
3938 interface(CONST_INTER);
3939 %}
3940
3941 // 32 bit unit decrement
3942 operand immI_M1()
3943 %{
3944 predicate(n->get_int() == -1);
3945 match(ConI);
3946
3947 op_cost(0);
3948 format %{ %}
3949 interface(CONST_INTER);
3950 %}
3951
3952 // Shift values for add/sub extension shift
3953 operand immIExt()
3954 %{
3955 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3956 match(ConI);
3957
3958 op_cost(0);
3959 format %{ %}
3960 interface(CONST_INTER);
3961 %}
3962
3963 operand immI_gt_1()
3964 %{
3965 predicate(n->get_int() > 1);
3966 match(ConI);
3967
3968 op_cost(0);
3969 format %{ %}
3970 interface(CONST_INTER);
3971 %}
3972
3973 operand immI_le_4()
3974 %{
3975 predicate(n->get_int() <= 4);
3976 match(ConI);
3977
3978 op_cost(0);
3979 format %{ %}
3980 interface(CONST_INTER);
3981 %}
3982
3983 operand immI_16()
3984 %{
3985 predicate(n->get_int() == 16);
3986 match(ConI);
3987
3988 op_cost(0);
3989 format %{ %}
3990 interface(CONST_INTER);
3991 %}
3992
3993 operand immI_24()
3994 %{
3995 predicate(n->get_int() == 24);
3996 match(ConI);
3997
3998 op_cost(0);
3999 format %{ %}
4000 interface(CONST_INTER);
4001 %}
4002
4003 operand immI_32()
4004 %{
4005 predicate(n->get_int() == 32);
4006 match(ConI);
4007
4008 op_cost(0);
4009 format %{ %}
4010 interface(CONST_INTER);
4011 %}
4012
4013 operand immI_48()
4014 %{
4015 predicate(n->get_int() == 48);
4016 match(ConI);
4017
4018 op_cost(0);
4019 format %{ %}
4020 interface(CONST_INTER);
4021 %}
4022
4023 operand immI_56()
4024 %{
4025 predicate(n->get_int() == 56);
4026 match(ConI);
4027
4028 op_cost(0);
4029 format %{ %}
4030 interface(CONST_INTER);
4031 %}
4032
4033 operand immI_255()
4034 %{
4035 predicate(n->get_int() == 255);
4036 match(ConI);
4037
4038 op_cost(0);
4039 format %{ %}
4040 interface(CONST_INTER);
4041 %}
4042
4043 operand immI_65535()
4044 %{
4045 predicate(n->get_int() == 65535);
4046 match(ConI);
4047
4048 op_cost(0);
4049 format %{ %}
4050 interface(CONST_INTER);
4051 %}
4052
4053 operand immI_positive()
4054 %{
4055 predicate(n->get_int() > 0);
4056 match(ConI);
4057
4058 op_cost(0);
4059 format %{ %}
4060 interface(CONST_INTER);
4061 %}
4062
4063 // BoolTest condition for signed compare
4064 operand immI_cmp_cond()
4065 %{
4066 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4067 match(ConI);
4068
4069 op_cost(0);
4070 format %{ %}
4071 interface(CONST_INTER);
4072 %}
4073
4074 // BoolTest condition for unsigned compare
4075 operand immI_cmpU_cond()
4076 %{
4077 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4078 match(ConI);
4079
4080 op_cost(0);
4081 format %{ %}
4082 interface(CONST_INTER);
4083 %}
4084
4085 operand immL_255()
4086 %{
4087 predicate(n->get_long() == 255L);
4088 match(ConL);
4089
4090 op_cost(0);
4091 format %{ %}
4092 interface(CONST_INTER);
4093 %}
4094
4095 operand immL_65535()
4096 %{
4097 predicate(n->get_long() == 65535L);
4098 match(ConL);
4099
4100 op_cost(0);
4101 format %{ %}
4102 interface(CONST_INTER);
4103 %}
4104
4105 operand immL_4294967295()
4106 %{
4107 predicate(n->get_long() == 4294967295L);
4108 match(ConL);
4109
4110 op_cost(0);
4111 format %{ %}
4112 interface(CONST_INTER);
4113 %}
4114
4115 operand immL_bitmask()
4116 %{
4117 predicate((n->get_long() != 0)
4118 && ((n->get_long() & 0xc000000000000000l) == 0)
4119 && is_power_of_2(n->get_long() + 1));
4120 match(ConL);
4121
4122 op_cost(0);
4123 format %{ %}
4124 interface(CONST_INTER);
4125 %}
4126
4127 operand immI_bitmask()
4128 %{
4129 predicate((n->get_int() != 0)
4130 && ((n->get_int() & 0xc0000000) == 0)
4131 && is_power_of_2(n->get_int() + 1));
4132 match(ConI);
4133
4134 op_cost(0);
4135 format %{ %}
4136 interface(CONST_INTER);
4137 %}
4138
4139 operand immL_positive_bitmaskI()
4140 %{
4141 predicate((n->get_long() != 0)
4142 && ((julong)n->get_long() < 0x80000000ULL)
4143 && is_power_of_2(n->get_long() + 1));
4144 match(ConL);
4145
4146 op_cost(0);
4147 format %{ %}
4148 interface(CONST_INTER);
4149 %}
4150
4151 // Scale values for scaled offset addressing modes (up to long but not quad)
4152 operand immIScale()
4153 %{
4154 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4155 match(ConI);
4156
4157 op_cost(0);
4158 format %{ %}
4159 interface(CONST_INTER);
4160 %}
4161
4162 // 5 bit signed integer
4163 operand immI5()
4164 %{
4165 predicate(Assembler::is_simm(n->get_int(), 5));
4166 match(ConI);
4167
4168 op_cost(0);
4169 format %{ %}
4170 interface(CONST_INTER);
4171 %}
4172
4173 // 7 bit unsigned integer
4174 operand immIU7()
4175 %{
4176 predicate(Assembler::is_uimm(n->get_int(), 7));
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 // Offset for scaled or unscaled immediate loads and stores
4185 operand immIOffset()
4186 %{
4187 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4188 match(ConI);
4189
4190 op_cost(0);
4191 format %{ %}
4192 interface(CONST_INTER);
4193 %}
4194
4195 operand immIOffset1()
4196 %{
4197 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4198 match(ConI);
4199
4200 op_cost(0);
4201 format %{ %}
4202 interface(CONST_INTER);
4203 %}
4204
4205 operand immIOffset2()
4206 %{
4207 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4208 match(ConI);
4209
4210 op_cost(0);
4211 format %{ %}
4212 interface(CONST_INTER);
4213 %}
4214
4215 operand immIOffset4()
4216 %{
4217 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4218 match(ConI);
4219
4220 op_cost(0);
4221 format %{ %}
4222 interface(CONST_INTER);
4223 %}
4224
4225 operand immIOffset8()
4226 %{
4227 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4228 match(ConI);
4229
4230 op_cost(0);
4231 format %{ %}
4232 interface(CONST_INTER);
4233 %}
4234
4235 operand immIOffset16()
4236 %{
4237 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4238 match(ConI);
4239
4240 op_cost(0);
4241 format %{ %}
4242 interface(CONST_INTER);
4243 %}
4244
4245 operand immLOffset()
4246 %{
4247 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4248 match(ConL);
4249
4250 op_cost(0);
4251 format %{ %}
4252 interface(CONST_INTER);
4253 %}
4254
4255 operand immLoffset1()
4256 %{
4257 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4258 match(ConL);
4259
4260 op_cost(0);
4261 format %{ %}
4262 interface(CONST_INTER);
4263 %}
4264
4265 operand immLoffset2()
4266 %{
4267 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4268 match(ConL);
4269
4270 op_cost(0);
4271 format %{ %}
4272 interface(CONST_INTER);
4273 %}
4274
4275 operand immLoffset4()
4276 %{
4277 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4278 match(ConL);
4279
4280 op_cost(0);
4281 format %{ %}
4282 interface(CONST_INTER);
4283 %}
4284
4285 operand immLoffset8()
4286 %{
4287 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4288 match(ConL);
4289
4290 op_cost(0);
4291 format %{ %}
4292 interface(CONST_INTER);
4293 %}
4294
4295 operand immLoffset16()
4296 %{
4297 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4298 match(ConL);
4299
4300 op_cost(0);
4301 format %{ %}
4302 interface(CONST_INTER);
4303 %}
4304
4305 // 5 bit signed long integer
4306 operand immL5()
4307 %{
4308 predicate(Assembler::is_simm(n->get_long(), 5));
4309 match(ConL);
4310
4311 op_cost(0);
4312 format %{ %}
4313 interface(CONST_INTER);
4314 %}
4315
4316 // 7 bit unsigned long integer
4317 operand immLU7()
4318 %{
4319 predicate(Assembler::is_uimm(n->get_long(), 7));
4320 match(ConL);
4321
4322 op_cost(0);
4323 format %{ %}
4324 interface(CONST_INTER);
4325 %}
4326
4327 // 8 bit signed value.
4328 operand immI8()
4329 %{
4330 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4331 match(ConI);
4332
4333 op_cost(0);
4334 format %{ %}
4335 interface(CONST_INTER);
4336 %}
4337
4338 // 8 bit signed value (simm8), or #simm8 LSL 8.
4339 operand immIDupV()
4340 %{
4341 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4342 match(ConI);
4343
4344 op_cost(0);
4345 format %{ %}
4346 interface(CONST_INTER);
4347 %}
4348
4349 // 8 bit signed value (simm8), or #simm8 LSL 8.
4350 operand immLDupV()
4351 %{
4352 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4353 match(ConL);
4354
4355 op_cost(0);
4356 format %{ %}
4357 interface(CONST_INTER);
4358 %}
4359
4360 // 8 bit signed value (simm8), or #simm8 LSL 8.
4361 operand immHDupV()
4362 %{
4363 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4364 match(ConH);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 // 8 bit integer valid for vector add sub immediate
4372 operand immBAddSubV()
4373 %{
4374 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4375 match(ConI);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 // 32 bit integer valid for add sub immediate
4383 operand immIAddSub()
4384 %{
4385 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4386 match(ConI);
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 // 32 bit integer valid for vector add sub immediate
4393 operand immIAddSubV()
4394 %{
4395 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4396 match(ConI);
4397
4398 op_cost(0);
4399 format %{ %}
4400 interface(CONST_INTER);
4401 %}
4402
4403 // 32 bit unsigned integer valid for logical immediate
4404
4405 operand immBLog()
4406 %{
4407 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4408 match(ConI);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 operand immSLog()
4416 %{
4417 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4418 match(ConI);
4419
4420 op_cost(0);
4421 format %{ %}
4422 interface(CONST_INTER);
4423 %}
4424
4425 operand immILog()
4426 %{
4427 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4428 match(ConI);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 // Integer operands 64 bit
4436 // 64 bit immediate
4437 operand immL()
4438 %{
4439 match(ConL);
4440
4441 op_cost(0);
4442 format %{ %}
4443 interface(CONST_INTER);
4444 %}
4445
4446 // 64 bit zero
4447 operand immL0()
4448 %{
4449 predicate(n->get_long() == 0);
4450 match(ConL);
4451
4452 op_cost(0);
4453 format %{ %}
4454 interface(CONST_INTER);
4455 %}
4456
4457 // 64 bit unit decrement
4458 operand immL_M1()
4459 %{
4460 predicate(n->get_long() == -1);
4461 match(ConL);
4462
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 // 64 bit integer valid for add sub immediate
4469 operand immLAddSub()
4470 %{
4471 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4472 match(ConL);
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // 64 bit integer valid for addv subv immediate
4479 operand immLAddSubV()
4480 %{
4481 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4482 match(ConL);
4483
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 // 64 bit integer valid for logical immediate
4490 operand immLLog()
4491 %{
4492 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4493 match(ConL);
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 // Long Immediate: low 32-bit mask
4500 operand immL_32bits()
4501 %{
4502 predicate(n->get_long() == 0xFFFFFFFFL);
4503 match(ConL);
4504 op_cost(0);
4505 format %{ %}
4506 interface(CONST_INTER);
4507 %}
4508
4509 // Pointer operands
4510 // Pointer Immediate
4511 operand immP()
4512 %{
4513 match(ConP);
4514
4515 op_cost(0);
4516 format %{ %}
4517 interface(CONST_INTER);
4518 %}
4519
4520 // nullptr Pointer Immediate
4521 operand immP0()
4522 %{
4523 predicate(n->get_ptr() == 0);
4524 match(ConP);
4525
4526 op_cost(0);
4527 format %{ %}
4528 interface(CONST_INTER);
4529 %}
4530
4531 // Pointer Immediate One
4532 // this is used in object initialization (initial object header)
4533 operand immP_1()
4534 %{
4535 predicate(n->get_ptr() == 1);
4536 match(ConP);
4537
4538 op_cost(0);
4539 format %{ %}
4540 interface(CONST_INTER);
4541 %}
4542
4543 // AOT Runtime Constants Address
4544 operand immAOTRuntimeConstantsAddress()
4545 %{
4546 // Check if the address is in the range of AOT Runtime Constants
4547 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4548 match(ConP);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 // Float and Double operands
4556 // Double Immediate
4557 operand immD()
4558 %{
4559 match(ConD);
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // Double Immediate: +0.0d
4566 operand immD0()
4567 %{
4568 predicate(jlong_cast(n->getd()) == 0);
4569 match(ConD);
4570
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // constant 'double +0.0'.
4577 operand immDPacked()
4578 %{
4579 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4580 match(ConD);
4581 op_cost(0);
4582 format %{ %}
4583 interface(CONST_INTER);
4584 %}
4585
4586 // Float Immediate
4587 operand immF()
4588 %{
4589 match(ConF);
4590 op_cost(0);
4591 format %{ %}
4592 interface(CONST_INTER);
4593 %}
4594
4595 // Float Immediate: +0.0f.
4596 operand immF0()
4597 %{
4598 predicate(jint_cast(n->getf()) == 0);
4599 match(ConF);
4600
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Half Float (FP16) Immediate
4607 operand immH()
4608 %{
4609 match(ConH);
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 //
4616 operand immFPacked()
4617 %{
4618 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4619 match(ConF);
4620 op_cost(0);
4621 format %{ %}
4622 interface(CONST_INTER);
4623 %}
4624
4625 // Narrow pointer operands
4626 // Narrow Pointer Immediate
4627 operand immN()
4628 %{
4629 match(ConN);
4630
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // Narrow nullptr Pointer Immediate
4637 operand immN0()
4638 %{
4639 predicate(n->get_narrowcon() == 0);
4640 match(ConN);
4641
4642 op_cost(0);
4643 format %{ %}
4644 interface(CONST_INTER);
4645 %}
4646
4647 operand immNKlass()
4648 %{
4649 match(ConNKlass);
4650
4651 op_cost(0);
4652 format %{ %}
4653 interface(CONST_INTER);
4654 %}
4655
4656 // Integer 32 bit Register Operands
4657 // Integer 32 bitRegister (excludes SP)
4658 operand iRegI()
4659 %{
4660 constraint(ALLOC_IN_RC(any_reg32));
4661 match(RegI);
4662 match(iRegINoSp);
4663 op_cost(0);
4664 format %{ %}
4665 interface(REG_INTER);
4666 %}
4667
4668 // Integer 32 bit Register not Special
4669 operand iRegINoSp()
4670 %{
4671 constraint(ALLOC_IN_RC(no_special_reg32));
4672 match(RegI);
4673 op_cost(0);
4674 format %{ %}
4675 interface(REG_INTER);
4676 %}
4677
4678 // Integer 64 bit Register Operands
4679 // Integer 64 bit Register (includes SP)
4680 operand iRegL()
4681 %{
4682 constraint(ALLOC_IN_RC(any_reg));
4683 match(RegL);
4684 match(iRegLNoSp);
4685 op_cost(0);
4686 format %{ %}
4687 interface(REG_INTER);
4688 %}
4689
4690 // Integer 64 bit Register not Special
4691 operand iRegLNoSp()
4692 %{
4693 constraint(ALLOC_IN_RC(no_special_reg));
4694 match(RegL);
4695 match(iRegL_R0);
4696 format %{ %}
4697 interface(REG_INTER);
4698 %}
4699
4700 // Pointer Register Operands
4701 // Pointer Register
4702 operand iRegP()
4703 %{
4704 constraint(ALLOC_IN_RC(ptr_reg));
4705 match(RegP);
4706 match(iRegPNoSp);
4707 match(iRegP_R0);
4708 //match(iRegP_R2);
4709 //match(iRegP_R4);
4710 match(iRegP_R5);
4711 match(thread_RegP);
4712 op_cost(0);
4713 format %{ %}
4714 interface(REG_INTER);
4715 %}
4716
4717 // Pointer 64 bit Register not Special
4718 operand iRegPNoSp()
4719 %{
4720 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4721 match(RegP);
4722 // match(iRegP);
4723 // match(iRegP_R0);
4724 // match(iRegP_R2);
4725 // match(iRegP_R4);
4726 // match(iRegP_R5);
4727 // match(thread_RegP);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // This operand is not allowed to use rfp even if
4734 // rfp is not used to hold the frame pointer.
4735 operand iRegPNoSpNoRfp()
4736 %{
4737 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4738 match(RegP);
4739 match(iRegPNoSp);
4740 op_cost(0);
4741 format %{ %}
4742 interface(REG_INTER);
4743 %}
4744
4745 // Pointer 64 bit Register R0 only
4746 operand iRegP_R0()
4747 %{
4748 constraint(ALLOC_IN_RC(r0_reg));
4749 match(RegP);
4750 // match(iRegP);
4751 match(iRegPNoSp);
4752 op_cost(0);
4753 format %{ %}
4754 interface(REG_INTER);
4755 %}
4756
4757 // Pointer 64 bit Register R1 only
4758 operand iRegP_R1()
4759 %{
4760 constraint(ALLOC_IN_RC(r1_reg));
4761 match(RegP);
4762 // match(iRegP);
4763 match(iRegPNoSp);
4764 op_cost(0);
4765 format %{ %}
4766 interface(REG_INTER);
4767 %}
4768
4769 // Pointer 64 bit Register R2 only
4770 operand iRegP_R2()
4771 %{
4772 constraint(ALLOC_IN_RC(r2_reg));
4773 match(RegP);
4774 // match(iRegP);
4775 match(iRegPNoSp);
4776 op_cost(0);
4777 format %{ %}
4778 interface(REG_INTER);
4779 %}
4780
4781 // Pointer 64 bit Register R3 only
4782 operand iRegP_R3()
4783 %{
4784 constraint(ALLOC_IN_RC(r3_reg));
4785 match(RegP);
4786 // match(iRegP);
4787 match(iRegPNoSp);
4788 op_cost(0);
4789 format %{ %}
4790 interface(REG_INTER);
4791 %}
4792
4793 // Pointer 64 bit Register R4 only
4794 operand iRegP_R4()
4795 %{
4796 constraint(ALLOC_IN_RC(r4_reg));
4797 match(RegP);
4798 // match(iRegP);
4799 match(iRegPNoSp);
4800 op_cost(0);
4801 format %{ %}
4802 interface(REG_INTER);
4803 %}
4804
4805 // Pointer 64 bit Register R5 only
4806 operand iRegP_R5()
4807 %{
4808 constraint(ALLOC_IN_RC(r5_reg));
4809 match(RegP);
4810 // match(iRegP);
4811 match(iRegPNoSp);
4812 op_cost(0);
4813 format %{ %}
4814 interface(REG_INTER);
4815 %}
4816
4817 // Pointer 64 bit Register R10 only
4818 operand iRegP_R10()
4819 %{
4820 constraint(ALLOC_IN_RC(r10_reg));
4821 match(RegP);
4822 // match(iRegP);
4823 match(iRegPNoSp);
4824 op_cost(0);
4825 format %{ %}
4826 interface(REG_INTER);
4827 %}
4828
4829 // Long 64 bit Register R0 only
4830 operand iRegL_R0()
4831 %{
4832 constraint(ALLOC_IN_RC(r0_reg));
4833 match(RegL);
4834 match(iRegLNoSp);
4835 op_cost(0);
4836 format %{ %}
4837 interface(REG_INTER);
4838 %}
4839
4840 // Long 64 bit Register R11 only
4841 operand iRegL_R11()
4842 %{
4843 constraint(ALLOC_IN_RC(r11_reg));
4844 match(RegL);
4845 match(iRegLNoSp);
4846 op_cost(0);
4847 format %{ %}
4848 interface(REG_INTER);
4849 %}
4850
4851 // Register R0 only
4852 operand iRegI_R0()
4853 %{
4854 constraint(ALLOC_IN_RC(int_r0_reg));
4855 match(RegI);
4856 match(iRegINoSp);
4857 op_cost(0);
4858 format %{ %}
4859 interface(REG_INTER);
4860 %}
4861
4862 // Register R2 only
4863 operand iRegI_R2()
4864 %{
4865 constraint(ALLOC_IN_RC(int_r2_reg));
4866 match(RegI);
4867 match(iRegINoSp);
4868 op_cost(0);
4869 format %{ %}
4870 interface(REG_INTER);
4871 %}
4872
4873 // Register R3 only
4874 operand iRegI_R3()
4875 %{
4876 constraint(ALLOC_IN_RC(int_r3_reg));
4877 match(RegI);
4878 match(iRegINoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884
4885 // Register R4 only
4886 operand iRegI_R4()
4887 %{
4888 constraint(ALLOC_IN_RC(int_r4_reg));
4889 match(RegI);
4890 match(iRegINoSp);
4891 op_cost(0);
4892 format %{ %}
4893 interface(REG_INTER);
4894 %}
4895
4896
4897 // Pointer Register Operands
4898 // Narrow Pointer Register
4899 operand iRegN()
4900 %{
4901 constraint(ALLOC_IN_RC(any_reg32));
4902 match(RegN);
4903 match(iRegNNoSp);
4904 op_cost(0);
4905 format %{ %}
4906 interface(REG_INTER);
4907 %}
4908
4909 // Integer 64 bit Register not Special
4910 operand iRegNNoSp()
4911 %{
4912 constraint(ALLOC_IN_RC(no_special_reg32));
4913 match(RegN);
4914 op_cost(0);
4915 format %{ %}
4916 interface(REG_INTER);
4917 %}
4918
4919 // Float Register
4920 // Float register operands
4921 operand vRegF()
4922 %{
4923 constraint(ALLOC_IN_RC(float_reg));
4924 match(RegF);
4925
4926 op_cost(0);
4927 format %{ %}
4928 interface(REG_INTER);
4929 %}
4930
4931 // Double Register
4932 // Double register operands
4933 operand vRegD()
4934 %{
4935 constraint(ALLOC_IN_RC(double_reg));
4936 match(RegD);
4937
4938 op_cost(0);
4939 format %{ %}
4940 interface(REG_INTER);
4941 %}
4942
4943 // Generic vector class. This will be used for
4944 // all vector operands, including NEON and SVE.
4945 operand vReg()
4946 %{
4947 constraint(ALLOC_IN_RC(dynamic));
4948 match(VecA);
4949 match(VecD);
4950 match(VecX);
4951
4952 op_cost(0);
4953 format %{ %}
4954 interface(REG_INTER);
4955 %}
4956
4957 operand vReg_V10()
4958 %{
4959 constraint(ALLOC_IN_RC(v10_veca_reg));
4960 match(vReg);
4961
4962 op_cost(0);
4963 format %{ %}
4964 interface(REG_INTER);
4965 %}
4966
4967 operand vReg_V11()
4968 %{
4969 constraint(ALLOC_IN_RC(v11_veca_reg));
4970 match(vReg);
4971
4972 op_cost(0);
4973 format %{ %}
4974 interface(REG_INTER);
4975 %}
4976
4977 operand vReg_V12()
4978 %{
4979 constraint(ALLOC_IN_RC(v12_veca_reg));
4980 match(vReg);
4981
4982 op_cost(0);
4983 format %{ %}
4984 interface(REG_INTER);
4985 %}
4986
4987 operand vReg_V13()
4988 %{
4989 constraint(ALLOC_IN_RC(v13_veca_reg));
4990 match(vReg);
4991
4992 op_cost(0);
4993 format %{ %}
4994 interface(REG_INTER);
4995 %}
4996
4997 operand vReg_V17()
4998 %{
4999 constraint(ALLOC_IN_RC(v17_veca_reg));
5000 match(vReg);
5001
5002 op_cost(0);
5003 format %{ %}
5004 interface(REG_INTER);
5005 %}
5006
5007 operand vReg_V18()
5008 %{
5009 constraint(ALLOC_IN_RC(v18_veca_reg));
5010 match(vReg);
5011
5012 op_cost(0);
5013 format %{ %}
5014 interface(REG_INTER);
5015 %}
5016
5017 operand vReg_V23()
5018 %{
5019 constraint(ALLOC_IN_RC(v23_veca_reg));
5020 match(vReg);
5021
5022 op_cost(0);
5023 format %{ %}
5024 interface(REG_INTER);
5025 %}
5026
5027 operand vReg_V24()
5028 %{
5029 constraint(ALLOC_IN_RC(v24_veca_reg));
5030 match(vReg);
5031
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand vecA()
5038 %{
5039 constraint(ALLOC_IN_RC(vectora_reg));
5040 match(VecA);
5041
5042 op_cost(0);
5043 format %{ %}
5044 interface(REG_INTER);
5045 %}
5046
5047 operand vecD()
5048 %{
5049 constraint(ALLOC_IN_RC(vectord_reg));
5050 match(VecD);
5051
5052 op_cost(0);
5053 format %{ %}
5054 interface(REG_INTER);
5055 %}
5056
5057 operand vecX()
5058 %{
5059 constraint(ALLOC_IN_RC(vectorx_reg));
5060 match(VecX);
5061
5062 op_cost(0);
5063 format %{ %}
5064 interface(REG_INTER);
5065 %}
5066
5067 operand vRegD_V0()
5068 %{
5069 constraint(ALLOC_IN_RC(v0_reg));
5070 match(RegD);
5071 op_cost(0);
5072 format %{ %}
5073 interface(REG_INTER);
5074 %}
5075
5076 operand vRegD_V1()
5077 %{
5078 constraint(ALLOC_IN_RC(v1_reg));
5079 match(RegD);
5080 op_cost(0);
5081 format %{ %}
5082 interface(REG_INTER);
5083 %}
5084
5085 operand vRegD_V2()
5086 %{
5087 constraint(ALLOC_IN_RC(v2_reg));
5088 match(RegD);
5089 op_cost(0);
5090 format %{ %}
5091 interface(REG_INTER);
5092 %}
5093
5094 operand vRegD_V3()
5095 %{
5096 constraint(ALLOC_IN_RC(v3_reg));
5097 match(RegD);
5098 op_cost(0);
5099 format %{ %}
5100 interface(REG_INTER);
5101 %}
5102
5103 operand vRegD_V4()
5104 %{
5105 constraint(ALLOC_IN_RC(v4_reg));
5106 match(RegD);
5107 op_cost(0);
5108 format %{ %}
5109 interface(REG_INTER);
5110 %}
5111
5112 operand vRegD_V5()
5113 %{
5114 constraint(ALLOC_IN_RC(v5_reg));
5115 match(RegD);
5116 op_cost(0);
5117 format %{ %}
5118 interface(REG_INTER);
5119 %}
5120
5121 operand vRegD_V6()
5122 %{
5123 constraint(ALLOC_IN_RC(v6_reg));
5124 match(RegD);
5125 op_cost(0);
5126 format %{ %}
5127 interface(REG_INTER);
5128 %}
5129
5130 operand vRegD_V7()
5131 %{
5132 constraint(ALLOC_IN_RC(v7_reg));
5133 match(RegD);
5134 op_cost(0);
5135 format %{ %}
5136 interface(REG_INTER);
5137 %}
5138
5139 operand vRegD_V12()
5140 %{
5141 constraint(ALLOC_IN_RC(v12_reg));
5142 match(RegD);
5143 op_cost(0);
5144 format %{ %}
5145 interface(REG_INTER);
5146 %}
5147
5148 operand vRegD_V13()
5149 %{
5150 constraint(ALLOC_IN_RC(v13_reg));
5151 match(RegD);
5152 op_cost(0);
5153 format %{ %}
5154 interface(REG_INTER);
5155 %}
5156
5157 operand pReg()
5158 %{
5159 constraint(ALLOC_IN_RC(pr_reg));
5160 match(RegVectMask);
5161 match(pRegGov);
5162 op_cost(0);
5163 format %{ %}
5164 interface(REG_INTER);
5165 %}
5166
5167 operand pRegGov()
5168 %{
5169 constraint(ALLOC_IN_RC(gov_pr));
5170 match(RegVectMask);
5171 match(pReg);
5172 op_cost(0);
5173 format %{ %}
5174 interface(REG_INTER);
5175 %}
5176
5177 operand pRegGov_P0()
5178 %{
5179 constraint(ALLOC_IN_RC(p0_reg));
5180 match(RegVectMask);
5181 op_cost(0);
5182 format %{ %}
5183 interface(REG_INTER);
5184 %}
5185
5186 operand pRegGov_P1()
5187 %{
5188 constraint(ALLOC_IN_RC(p1_reg));
5189 match(RegVectMask);
5190 op_cost(0);
5191 format %{ %}
5192 interface(REG_INTER);
5193 %}
5194
5195 // Flags register, used as output of signed compare instructions
5196
5197 // note that on AArch64 we also use this register as the output for
5198 // for floating point compare instructions (CmpF CmpD). this ensures
5199 // that ordered inequality tests use GT, GE, LT or LE none of which
5200 // pass through cases where the result is unordered i.e. one or both
5201 // inputs to the compare is a NaN. this means that the ideal code can
5202 // replace e.g. a GT with an LE and not end up capturing the NaN case
5203 // (where the comparison should always fail). EQ and NE tests are
5204 // always generated in ideal code so that unordered folds into the NE
5205 // case, matching the behaviour of AArch64 NE.
5206 //
5207 // This differs from x86 where the outputs of FP compares use a
5208 // special FP flags registers and where compares based on this
5209 // register are distinguished into ordered inequalities (cmpOpUCF) and
5210 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5211 // to explicitly handle the unordered case in branches. x86 also has
5212 // to include extra CMoveX rules to accept a cmpOpUCF input.
5213
5214 operand rFlagsReg()
5215 %{
5216 constraint(ALLOC_IN_RC(int_flags));
5217 match(RegFlags);
5218
5219 op_cost(0);
5220 format %{ "RFLAGS" %}
5221 interface(REG_INTER);
5222 %}
5223
5224 // Flags register, used as output of unsigned compare instructions
5225 operand rFlagsRegU()
5226 %{
5227 constraint(ALLOC_IN_RC(int_flags));
5228 match(RegFlags);
5229
5230 op_cost(0);
5231 format %{ "RFLAGSU" %}
5232 interface(REG_INTER);
5233 %}
5234
5235 // Special Registers
5236
5237 // Method Register
5238 operand inline_cache_RegP(iRegP reg)
5239 %{
5240 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5241 match(reg);
5242 match(iRegPNoSp);
5243 op_cost(0);
5244 format %{ %}
5245 interface(REG_INTER);
5246 %}
5247
5248 // Thread Register
5249 operand thread_RegP(iRegP reg)
5250 %{
5251 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5252 match(reg);
5253 op_cost(0);
5254 format %{ %}
5255 interface(REG_INTER);
5256 %}
5257
5258 //----------Memory Operands----------------------------------------------------
5259
5260 operand indirect(iRegP reg)
5261 %{
5262 constraint(ALLOC_IN_RC(ptr_reg));
5263 match(reg);
5264 op_cost(0);
5265 format %{ "[$reg]" %}
5266 interface(MEMORY_INTER) %{
5267 base($reg);
5268 index(0xffffffff);
5269 scale(0x0);
5270 disp(0x0);
5271 %}
5272 %}
5273
5274 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5275 %{
5276 constraint(ALLOC_IN_RC(ptr_reg));
5277 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5278 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5279 op_cost(0);
5280 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5281 interface(MEMORY_INTER) %{
5282 base($reg);
5283 index($ireg);
5284 scale($scale);
5285 disp(0x0);
5286 %}
5287 %}
5288
5289 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5293 match(AddP reg (LShiftL lreg scale));
5294 op_cost(0);
5295 format %{ "$reg, $lreg lsl($scale)" %}
5296 interface(MEMORY_INTER) %{
5297 base($reg);
5298 index($lreg);
5299 scale($scale);
5300 disp(0x0);
5301 %}
5302 %}
5303
5304 operand indIndexI2L(iRegP reg, iRegI ireg)
5305 %{
5306 constraint(ALLOC_IN_RC(ptr_reg));
5307 match(AddP reg (ConvI2L ireg));
5308 op_cost(0);
5309 format %{ "$reg, $ireg, 0, I2L" %}
5310 interface(MEMORY_INTER) %{
5311 base($reg);
5312 index($ireg);
5313 scale(0x0);
5314 disp(0x0);
5315 %}
5316 %}
5317
5318 operand indIndex(iRegP reg, iRegL lreg)
5319 %{
5320 constraint(ALLOC_IN_RC(ptr_reg));
5321 match(AddP reg lreg);
5322 op_cost(0);
5323 format %{ "$reg, $lreg" %}
5324 interface(MEMORY_INTER) %{
5325 base($reg);
5326 index($lreg);
5327 scale(0x0);
5328 disp(0x0);
5329 %}
5330 %}
5331
5332 operand indOffI1(iRegP reg, immIOffset1 off)
5333 %{
5334 constraint(ALLOC_IN_RC(ptr_reg));
5335 match(AddP reg off);
5336 op_cost(0);
5337 format %{ "[$reg, $off]" %}
5338 interface(MEMORY_INTER) %{
5339 base($reg);
5340 index(0xffffffff);
5341 scale(0x0);
5342 disp($off);
5343 %}
5344 %}
5345
5346 operand indOffI2(iRegP reg, immIOffset2 off)
5347 %{
5348 constraint(ALLOC_IN_RC(ptr_reg));
5349 match(AddP reg off);
5350 op_cost(0);
5351 format %{ "[$reg, $off]" %}
5352 interface(MEMORY_INTER) %{
5353 base($reg);
5354 index(0xffffffff);
5355 scale(0x0);
5356 disp($off);
5357 %}
5358 %}
5359
5360 operand indOffI4(iRegP reg, immIOffset4 off)
5361 %{
5362 constraint(ALLOC_IN_RC(ptr_reg));
5363 match(AddP reg off);
5364 op_cost(0);
5365 format %{ "[$reg, $off]" %}
5366 interface(MEMORY_INTER) %{
5367 base($reg);
5368 index(0xffffffff);
5369 scale(0x0);
5370 disp($off);
5371 %}
5372 %}
5373
5374 operand indOffI8(iRegP reg, immIOffset8 off)
5375 %{
5376 constraint(ALLOC_IN_RC(ptr_reg));
5377 match(AddP reg off);
5378 op_cost(0);
5379 format %{ "[$reg, $off]" %}
5380 interface(MEMORY_INTER) %{
5381 base($reg);
5382 index(0xffffffff);
5383 scale(0x0);
5384 disp($off);
5385 %}
5386 %}
5387
5388 operand indOffI16(iRegP reg, immIOffset16 off)
5389 %{
5390 constraint(ALLOC_IN_RC(ptr_reg));
5391 match(AddP reg off);
5392 op_cost(0);
5393 format %{ "[$reg, $off]" %}
5394 interface(MEMORY_INTER) %{
5395 base($reg);
5396 index(0xffffffff);
5397 scale(0x0);
5398 disp($off);
5399 %}
5400 %}
5401
5402 operand indOffL1(iRegP reg, immLoffset1 off)
5403 %{
5404 constraint(ALLOC_IN_RC(ptr_reg));
5405 match(AddP reg off);
5406 op_cost(0);
5407 format %{ "[$reg, $off]" %}
5408 interface(MEMORY_INTER) %{
5409 base($reg);
5410 index(0xffffffff);
5411 scale(0x0);
5412 disp($off);
5413 %}
5414 %}
5415
5416 operand indOffL2(iRegP reg, immLoffset2 off)
5417 %{
5418 constraint(ALLOC_IN_RC(ptr_reg));
5419 match(AddP reg off);
5420 op_cost(0);
5421 format %{ "[$reg, $off]" %}
5422 interface(MEMORY_INTER) %{
5423 base($reg);
5424 index(0xffffffff);
5425 scale(0x0);
5426 disp($off);
5427 %}
5428 %}
5429
5430 operand indOffL4(iRegP reg, immLoffset4 off)
5431 %{
5432 constraint(ALLOC_IN_RC(ptr_reg));
5433 match(AddP reg off);
5434 op_cost(0);
5435 format %{ "[$reg, $off]" %}
5436 interface(MEMORY_INTER) %{
5437 base($reg);
5438 index(0xffffffff);
5439 scale(0x0);
5440 disp($off);
5441 %}
5442 %}
5443
5444 operand indOffL8(iRegP reg, immLoffset8 off)
5445 %{
5446 constraint(ALLOC_IN_RC(ptr_reg));
5447 match(AddP reg off);
5448 op_cost(0);
5449 format %{ "[$reg, $off]" %}
5450 interface(MEMORY_INTER) %{
5451 base($reg);
5452 index(0xffffffff);
5453 scale(0x0);
5454 disp($off);
5455 %}
5456 %}
5457
5458 operand indOffL16(iRegP reg, immLoffset16 off)
5459 %{
5460 constraint(ALLOC_IN_RC(ptr_reg));
5461 match(AddP reg off);
5462 op_cost(0);
5463 format %{ "[$reg, $off]" %}
5464 interface(MEMORY_INTER) %{
5465 base($reg);
5466 index(0xffffffff);
5467 scale(0x0);
5468 disp($off);
5469 %}
5470 %}
5471
5472 operand indirectX2P(iRegL reg)
5473 %{
5474 constraint(ALLOC_IN_RC(ptr_reg));
5475 match(CastX2P reg);
5476 op_cost(0);
5477 format %{ "[$reg]\t# long -> ptr" %}
5478 interface(MEMORY_INTER) %{
5479 base($reg);
5480 index(0xffffffff);
5481 scale(0x0);
5482 disp(0x0);
5483 %}
5484 %}
5485
5486 operand indOffX2P(iRegL reg, immLOffset off)
5487 %{
5488 constraint(ALLOC_IN_RC(ptr_reg));
5489 match(AddP (CastX2P reg) off);
5490 op_cost(0);
5491 format %{ "[$reg, $off]\t# long -> ptr" %}
5492 interface(MEMORY_INTER) %{
5493 base($reg);
5494 index(0xffffffff);
5495 scale(0x0);
5496 disp($off);
5497 %}
5498 %}
5499
5500 operand indirectN(iRegN reg)
5501 %{
5502 predicate(CompressedOops::shift() == 0);
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(DecodeN reg);
5505 op_cost(0);
5506 format %{ "[$reg]\t# narrow" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index(0xffffffff);
5510 scale(0x0);
5511 disp(0x0);
5512 %}
5513 %}
5514
5515 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5516 %{
5517 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5518 constraint(ALLOC_IN_RC(ptr_reg));
5519 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5520 op_cost(0);
5521 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5522 interface(MEMORY_INTER) %{
5523 base($reg);
5524 index($ireg);
5525 scale($scale);
5526 disp(0x0);
5527 %}
5528 %}
5529
5530 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5531 %{
5532 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5533 constraint(ALLOC_IN_RC(ptr_reg));
5534 match(AddP (DecodeN reg) (LShiftL lreg scale));
5535 op_cost(0);
5536 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5537 interface(MEMORY_INTER) %{
5538 base($reg);
5539 index($lreg);
5540 scale($scale);
5541 disp(0x0);
5542 %}
5543 %}
5544
5545 operand indIndexI2LN(iRegN reg, iRegI ireg)
5546 %{
5547 predicate(CompressedOops::shift() == 0);
5548 constraint(ALLOC_IN_RC(ptr_reg));
5549 match(AddP (DecodeN reg) (ConvI2L ireg));
5550 op_cost(0);
5551 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5552 interface(MEMORY_INTER) %{
5553 base($reg);
5554 index($ireg);
5555 scale(0x0);
5556 disp(0x0);
5557 %}
5558 %}
5559
5560 operand indIndexN(iRegN reg, iRegL lreg)
5561 %{
5562 predicate(CompressedOops::shift() == 0);
5563 constraint(ALLOC_IN_RC(ptr_reg));
5564 match(AddP (DecodeN reg) lreg);
5565 op_cost(0);
5566 format %{ "$reg, $lreg\t# narrow" %}
5567 interface(MEMORY_INTER) %{
5568 base($reg);
5569 index($lreg);
5570 scale(0x0);
5571 disp(0x0);
5572 %}
5573 %}
5574
5575 operand indOffIN(iRegN reg, immIOffset off)
5576 %{
5577 predicate(CompressedOops::shift() == 0);
5578 constraint(ALLOC_IN_RC(ptr_reg));
5579 match(AddP (DecodeN reg) off);
5580 op_cost(0);
5581 format %{ "[$reg, $off]\t# narrow" %}
5582 interface(MEMORY_INTER) %{
5583 base($reg);
5584 index(0xffffffff);
5585 scale(0x0);
5586 disp($off);
5587 %}
5588 %}
5589
5590 operand indOffLN(iRegN reg, immLOffset off)
5591 %{
5592 predicate(CompressedOops::shift() == 0);
5593 constraint(ALLOC_IN_RC(ptr_reg));
5594 match(AddP (DecodeN reg) off);
5595 op_cost(0);
5596 format %{ "[$reg, $off]\t# narrow" %}
5597 interface(MEMORY_INTER) %{
5598 base($reg);
5599 index(0xffffffff);
5600 scale(0x0);
5601 disp($off);
5602 %}
5603 %}
5604
5605
5606 //----------Special Memory Operands--------------------------------------------
5607 // Stack Slot Operand - This operand is used for loading and storing temporary
5608 // values on the stack where a match requires a value to
5609 // flow through memory.
5610 operand stackSlotP(sRegP reg)
5611 %{
5612 constraint(ALLOC_IN_RC(stack_slots));
5613 op_cost(100);
5614 // No match rule because this operand is only generated in matching
5615 // match(RegP);
5616 format %{ "[$reg]" %}
5617 interface(MEMORY_INTER) %{
5618 base(0x1e); // RSP
5619 index(0x0); // No Index
5620 scale(0x0); // No Scale
5621 disp($reg); // Stack Offset
5622 %}
5623 %}
5624
5625 operand stackSlotI(sRegI reg)
5626 %{
5627 constraint(ALLOC_IN_RC(stack_slots));
5628 // No match rule because this operand is only generated in matching
5629 // match(RegI);
5630 format %{ "[$reg]" %}
5631 interface(MEMORY_INTER) %{
5632 base(0x1e); // RSP
5633 index(0x0); // No Index
5634 scale(0x0); // No Scale
5635 disp($reg); // Stack Offset
5636 %}
5637 %}
5638
5639 operand stackSlotF(sRegF reg)
5640 %{
5641 constraint(ALLOC_IN_RC(stack_slots));
5642 // No match rule because this operand is only generated in matching
5643 // match(RegF);
5644 format %{ "[$reg]" %}
5645 interface(MEMORY_INTER) %{
5646 base(0x1e); // RSP
5647 index(0x0); // No Index
5648 scale(0x0); // No Scale
5649 disp($reg); // Stack Offset
5650 %}
5651 %}
5652
5653 operand stackSlotD(sRegD reg)
5654 %{
5655 constraint(ALLOC_IN_RC(stack_slots));
5656 // No match rule because this operand is only generated in matching
5657 // match(RegD);
5658 format %{ "[$reg]" %}
5659 interface(MEMORY_INTER) %{
5660 base(0x1e); // RSP
5661 index(0x0); // No Index
5662 scale(0x0); // No Scale
5663 disp($reg); // Stack Offset
5664 %}
5665 %}
5666
5667 operand stackSlotL(sRegL reg)
5668 %{
5669 constraint(ALLOC_IN_RC(stack_slots));
5670 // No match rule because this operand is only generated in matching
5671 // match(RegL);
5672 format %{ "[$reg]" %}
5673 interface(MEMORY_INTER) %{
5674 base(0x1e); // RSP
5675 index(0x0); // No Index
5676 scale(0x0); // No Scale
5677 disp($reg); // Stack Offset
5678 %}
5679 %}
5680
5681 // Operands for expressing Control Flow
5682 // NOTE: Label is a predefined operand which should not be redefined in
5683 // the AD file. It is generically handled within the ADLC.
5684
5685 //----------Conditional Branch Operands----------------------------------------
5686 // Comparison Op - This is the operation of the comparison, and is limited to
5687 // the following set of codes:
5688 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5689 //
5690 // Other attributes of the comparison, such as unsignedness, are specified
5691 // by the comparison instruction that sets a condition code flags register.
5692 // That result is represented by a flags operand whose subtype is appropriate
5693 // to the unsignedness (etc.) of the comparison.
5694 //
5695 // Later, the instruction which matches both the Comparison Op (a Bool) and
5696 // the flags (produced by the Cmp) specifies the coding of the comparison op
5697 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5698
5699 // used for signed integral comparisons and fp comparisons
5700
5701 operand cmpOp()
5702 %{
5703 match(Bool);
5704
5705 format %{ "" %}
5706 interface(COND_INTER) %{
5707 equal(0x0, "eq");
5708 not_equal(0x1, "ne");
5709 less(0xb, "lt");
5710 greater_equal(0xa, "ge");
5711 less_equal(0xd, "le");
5712 greater(0xc, "gt");
5713 overflow(0x6, "vs");
5714 no_overflow(0x7, "vc");
5715 %}
5716 %}
5717
5718 // used for unsigned integral comparisons
5719
5720 operand cmpOpU()
5721 %{
5722 match(Bool);
5723
5724 format %{ "" %}
5725 interface(COND_INTER) %{
5726 equal(0x0, "eq");
5727 not_equal(0x1, "ne");
5728 less(0x3, "lo");
5729 greater_equal(0x2, "hs");
5730 less_equal(0x9, "ls");
5731 greater(0x8, "hi");
5732 overflow(0x6, "vs");
5733 no_overflow(0x7, "vc");
5734 %}
5735 %}
5736
5737 // used for certain integral comparisons which can be
5738 // converted to cbxx or tbxx instructions
5739
5740 operand cmpOpEqNe()
5741 %{
5742 match(Bool);
5743 op_cost(0);
5744 predicate(n->as_Bool()->_test._test == BoolTest::ne
5745 || n->as_Bool()->_test._test == BoolTest::eq);
5746
5747 format %{ "" %}
5748 interface(COND_INTER) %{
5749 equal(0x0, "eq");
5750 not_equal(0x1, "ne");
5751 less(0xb, "lt");
5752 greater_equal(0xa, "ge");
5753 less_equal(0xd, "le");
5754 greater(0xc, "gt");
5755 overflow(0x6, "vs");
5756 no_overflow(0x7, "vc");
5757 %}
5758 %}
5759
5760 // used for certain integral comparisons which can be
5761 // converted to cbxx or tbxx instructions
5762
5763 operand cmpOpLtGe()
5764 %{
5765 match(Bool);
5766 op_cost(0);
5767
5768 predicate(n->as_Bool()->_test._test == BoolTest::lt
5769 || n->as_Bool()->_test._test == BoolTest::ge);
5770
5771 format %{ "" %}
5772 interface(COND_INTER) %{
5773 equal(0x0, "eq");
5774 not_equal(0x1, "ne");
5775 less(0xb, "lt");
5776 greater_equal(0xa, "ge");
5777 less_equal(0xd, "le");
5778 greater(0xc, "gt");
5779 overflow(0x6, "vs");
5780 no_overflow(0x7, "vc");
5781 %}
5782 %}
5783
5784 // used for certain unsigned integral comparisons which can be
5785 // converted to cbxx or tbxx instructions
5786
5787 operand cmpOpUEqNeLeGt()
5788 %{
5789 match(Bool);
5790 op_cost(0);
5791
5792 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5793 n->as_Bool()->_test._test == BoolTest::ne ||
5794 n->as_Bool()->_test._test == BoolTest::le ||
5795 n->as_Bool()->_test._test == BoolTest::gt);
5796
5797 format %{ "" %}
5798 interface(COND_INTER) %{
5799 equal(0x0, "eq");
5800 not_equal(0x1, "ne");
5801 less(0x3, "lo");
5802 greater_equal(0x2, "hs");
5803 less_equal(0x9, "ls");
5804 greater(0x8, "hi");
5805 overflow(0x6, "vs");
5806 no_overflow(0x7, "vc");
5807 %}
5808 %}
5809
5810 // Special operand allowing long args to int ops to be truncated for free
5811
5812 operand iRegL2I(iRegL reg) %{
5813
5814 op_cost(0);
5815
5816 match(ConvL2I reg);
5817
5818 format %{ "l2i($reg)" %}
5819
5820 interface(REG_INTER)
5821 %}
5822
5823 operand iRegL2P(iRegL reg) %{
5824
5825 op_cost(0);
5826
5827 match(CastX2P reg);
5828
5829 format %{ "l2p($reg)" %}
5830
5831 interface(REG_INTER)
5832 %}
5833
5834 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5835 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5836 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5837 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5838
5839 //----------OPERAND CLASSES----------------------------------------------------
5840 // Operand Classes are groups of operands that are used as to simplify
5841 // instruction definitions by not requiring the AD writer to specify
5842 // separate instructions for every form of operand when the
5843 // instruction accepts multiple operand types with the same basic
5844 // encoding and format. The classic case of this is memory operands.
5845
5846 // memory is used to define read/write location for load/store
5847 // instruction defs. we can turn a memory op into an Address
5848
5849 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5850 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5851
5852 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5853 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5854
5855 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5856 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5857
5858 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5859 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5860
5861 // All of the memory operands. For the pipeline description.
5862 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5863 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5864 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5865
5866
5867 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5868 // operations. it allows the src to be either an iRegI or a (ConvL2I
5869 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5870 // can be elided because the 32-bit instruction will just employ the
5871 // lower 32 bits anyway.
5872 //
5873 // n.b. this does not elide all L2I conversions. if the truncated
5874 // value is consumed by more than one operation then the ConvL2I
5875 // cannot be bundled into the consuming nodes so an l2i gets planted
5876 // (actually a movw $dst $src) and the downstream instructions consume
5877 // the result of the l2i as an iRegI input. That's a shame since the
5878 // movw is actually redundant but its not too costly.
5879
5880 opclass iRegIorL2I(iRegI, iRegL2I);
5881 opclass iRegPorL2P(iRegP, iRegL2P);
5882
5883 //----------PIPELINE-----------------------------------------------------------
5884 // Rules which define the behavior of the target architectures pipeline.
5885
5886 // For specific pipelines, eg A53, define the stages of that pipeline
5887 //pipe_desc(ISS, EX1, EX2, WR);
5888 #define ISS S0
5889 #define EX1 S1
5890 #define EX2 S2
5891 #define WR S3
5892
5893 // Integer ALU reg operation
5894 pipeline %{
5895
5896 attributes %{
5897 // ARM instructions are of fixed length
5898 fixed_size_instructions; // Fixed size instructions TODO does
5899 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5900 // ARM instructions come in 32-bit word units
5901 instruction_unit_size = 4; // An instruction is 4 bytes long
5902 instruction_fetch_unit_size = 64; // The processor fetches one line
5903 instruction_fetch_units = 1; // of 64 bytes
5904 %}
5905
5906 // We don't use an actual pipeline model so don't care about resources
5907 // or description. we do use pipeline classes to introduce fixed
5908 // latencies
5909
5910 //----------RESOURCES----------------------------------------------------------
5911 // Resources are the functional units available to the machine
5912
5913 resources( INS0, INS1, INS01 = INS0 | INS1,
5914 ALU0, ALU1, ALU = ALU0 | ALU1,
5915 MAC,
5916 DIV,
5917 BRANCH,
5918 LDST,
5919 NEON_FP);
5920
5921 //----------PIPELINE DESCRIPTION-----------------------------------------------
5922 // Pipeline Description specifies the stages in the machine's pipeline
5923
5924 // Define the pipeline as a generic 6 stage pipeline
5925 pipe_desc(S0, S1, S2, S3, S4, S5);
5926
5927 //----------PIPELINE CLASSES---------------------------------------------------
5928 // Pipeline Classes describe the stages in which input and output are
5929 // referenced by the hardware pipeline.
5930
5931 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5932 %{
5933 single_instruction;
5934 src1 : S1(read);
5935 src2 : S2(read);
5936 dst : S5(write);
5937 INS01 : ISS;
5938 NEON_FP : S5;
5939 %}
5940
5941 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5942 %{
5943 single_instruction;
5944 src1 : S1(read);
5945 src2 : S2(read);
5946 dst : S5(write);
5947 INS01 : ISS;
5948 NEON_FP : S5;
5949 %}
5950
5951 pipe_class fp_uop_s(vRegF dst, vRegF src)
5952 %{
5953 single_instruction;
5954 src : S1(read);
5955 dst : S5(write);
5956 INS01 : ISS;
5957 NEON_FP : S5;
5958 %}
5959
5960 pipe_class fp_uop_d(vRegD dst, vRegD src)
5961 %{
5962 single_instruction;
5963 src : S1(read);
5964 dst : S5(write);
5965 INS01 : ISS;
5966 NEON_FP : S5;
5967 %}
5968
5969 pipe_class fp_d2f(vRegF dst, vRegD src)
5970 %{
5971 single_instruction;
5972 src : S1(read);
5973 dst : S5(write);
5974 INS01 : ISS;
5975 NEON_FP : S5;
5976 %}
5977
5978 pipe_class fp_f2d(vRegD dst, vRegF src)
5979 %{
5980 single_instruction;
5981 src : S1(read);
5982 dst : S5(write);
5983 INS01 : ISS;
5984 NEON_FP : S5;
5985 %}
5986
5987 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5988 %{
5989 single_instruction;
5990 src : S1(read);
5991 dst : S5(write);
5992 INS01 : ISS;
5993 NEON_FP : S5;
5994 %}
5995
5996 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5997 %{
5998 single_instruction;
5999 src : S1(read);
6000 dst : S5(write);
6001 INS01 : ISS;
6002 NEON_FP : S5;
6003 %}
6004
6005 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6006 %{
6007 single_instruction;
6008 src : S1(read);
6009 dst : S5(write);
6010 INS01 : ISS;
6011 NEON_FP : S5;
6012 %}
6013
6014 pipe_class fp_l2f(vRegF dst, iRegL src)
6015 %{
6016 single_instruction;
6017 src : S1(read);
6018 dst : S5(write);
6019 INS01 : ISS;
6020 NEON_FP : S5;
6021 %}
6022
6023 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6024 %{
6025 single_instruction;
6026 src : S1(read);
6027 dst : S5(write);
6028 INS01 : ISS;
6029 NEON_FP : S5;
6030 %}
6031
6032 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6033 %{
6034 single_instruction;
6035 src : S1(read);
6036 dst : S5(write);
6037 INS01 : ISS;
6038 NEON_FP : S5;
6039 %}
6040
6041 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6042 %{
6043 single_instruction;
6044 src : S1(read);
6045 dst : S5(write);
6046 INS01 : ISS;
6047 NEON_FP : S5;
6048 %}
6049
6050 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6051 %{
6052 single_instruction;
6053 src : S1(read);
6054 dst : S5(write);
6055 INS01 : ISS;
6056 NEON_FP : S5;
6057 %}
6058
6059 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6060 %{
6061 single_instruction;
6062 src1 : S1(read);
6063 src2 : S2(read);
6064 dst : S5(write);
6065 INS0 : ISS;
6066 NEON_FP : S5;
6067 %}
6068
6069 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6070 %{
6071 single_instruction;
6072 src1 : S1(read);
6073 src2 : S2(read);
6074 dst : S5(write);
6075 INS0 : ISS;
6076 NEON_FP : S5;
6077 %}
6078
6079 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6080 %{
6081 single_instruction;
6082 cr : S1(read);
6083 src1 : S1(read);
6084 src2 : S1(read);
6085 dst : S3(write);
6086 INS01 : ISS;
6087 NEON_FP : S3;
6088 %}
6089
6090 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6091 %{
6092 single_instruction;
6093 cr : S1(read);
6094 src1 : S1(read);
6095 src2 : S1(read);
6096 dst : S3(write);
6097 INS01 : ISS;
6098 NEON_FP : S3;
6099 %}
6100
6101 pipe_class fp_imm_s(vRegF dst)
6102 %{
6103 single_instruction;
6104 dst : S3(write);
6105 INS01 : ISS;
6106 NEON_FP : S3;
6107 %}
6108
6109 pipe_class fp_imm_d(vRegD dst)
6110 %{
6111 single_instruction;
6112 dst : S3(write);
6113 INS01 : ISS;
6114 NEON_FP : S3;
6115 %}
6116
6117 pipe_class fp_load_constant_s(vRegF dst)
6118 %{
6119 single_instruction;
6120 dst : S4(write);
6121 INS01 : ISS;
6122 NEON_FP : S4;
6123 %}
6124
6125 pipe_class fp_load_constant_d(vRegD dst)
6126 %{
6127 single_instruction;
6128 dst : S4(write);
6129 INS01 : ISS;
6130 NEON_FP : S4;
6131 %}
6132
6133 //------- Integer ALU operations --------------------------
6134
6135 // Integer ALU reg-reg operation
6136 // Operands needed in EX1, result generated in EX2
6137 // Eg. ADD x0, x1, x2
6138 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6139 %{
6140 single_instruction;
6141 dst : EX2(write);
6142 src1 : EX1(read);
6143 src2 : EX1(read);
6144 INS01 : ISS; // Dual issue as instruction 0 or 1
6145 ALU : EX2;
6146 %}
6147
6148 // Integer ALU reg-reg operation with constant shift
6149 // Shifted register must be available in LATE_ISS instead of EX1
6150 // Eg. ADD x0, x1, x2, LSL #2
6151 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6152 %{
6153 single_instruction;
6154 dst : EX2(write);
6155 src1 : EX1(read);
6156 src2 : ISS(read);
6157 INS01 : ISS;
6158 ALU : EX2;
6159 %}
6160
6161 // Integer ALU reg operation with constant shift
6162 // Eg. LSL x0, x1, #shift
6163 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6164 %{
6165 single_instruction;
6166 dst : EX2(write);
6167 src1 : ISS(read);
6168 INS01 : ISS;
6169 ALU : EX2;
6170 %}
6171
6172 // Integer ALU reg-reg operation with variable shift
6173 // Both operands must be available in LATE_ISS instead of EX1
6174 // Result is available in EX1 instead of EX2
6175 // Eg. LSLV x0, x1, x2
6176 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6177 %{
6178 single_instruction;
6179 dst : EX1(write);
6180 src1 : ISS(read);
6181 src2 : ISS(read);
6182 INS01 : ISS;
6183 ALU : EX1;
6184 %}
6185
6186 // Integer ALU reg-reg operation with extract
6187 // As for _vshift above, but result generated in EX2
6188 // Eg. EXTR x0, x1, x2, #N
6189 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6190 %{
6191 single_instruction;
6192 dst : EX2(write);
6193 src1 : ISS(read);
6194 src2 : ISS(read);
6195 INS1 : ISS; // Can only dual issue as Instruction 1
6196 ALU : EX1;
6197 %}
6198
6199 // Integer ALU reg operation
6200 // Eg. NEG x0, x1
6201 pipe_class ialu_reg(iRegI dst, iRegI src)
6202 %{
6203 single_instruction;
6204 dst : EX2(write);
6205 src : EX1(read);
6206 INS01 : ISS;
6207 ALU : EX2;
6208 %}
6209
6210 // Integer ALU reg mmediate operation
6211 // Eg. ADD x0, x1, #N
6212 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6213 %{
6214 single_instruction;
6215 dst : EX2(write);
6216 src1 : EX1(read);
6217 INS01 : ISS;
6218 ALU : EX2;
6219 %}
6220
6221 // Integer ALU immediate operation (no source operands)
6222 // Eg. MOV x0, #N
6223 pipe_class ialu_imm(iRegI dst)
6224 %{
6225 single_instruction;
6226 dst : EX1(write);
6227 INS01 : ISS;
6228 ALU : EX1;
6229 %}
6230
6231 //------- Compare operation -------------------------------
6232
6233 // Compare reg-reg
6234 // Eg. CMP x0, x1
6235 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6236 %{
6237 single_instruction;
6238 // fixed_latency(16);
6239 cr : EX2(write);
6240 op1 : EX1(read);
6241 op2 : EX1(read);
6242 INS01 : ISS;
6243 ALU : EX2;
6244 %}
6245
6246 // Compare reg-reg
6247 // Eg. CMP x0, #N
6248 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6249 %{
6250 single_instruction;
6251 // fixed_latency(16);
6252 cr : EX2(write);
6253 op1 : EX1(read);
6254 INS01 : ISS;
6255 ALU : EX2;
6256 %}
6257
6258 //------- Conditional instructions ------------------------
6259
6260 // Conditional no operands
6261 // Eg. CSINC x0, zr, zr, <cond>
6262 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6263 %{
6264 single_instruction;
6265 cr : EX1(read);
6266 dst : EX2(write);
6267 INS01 : ISS;
6268 ALU : EX2;
6269 %}
6270
6271 // Conditional 2 operand
6272 // EG. CSEL X0, X1, X2, <cond>
6273 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6274 %{
6275 single_instruction;
6276 cr : EX1(read);
6277 src1 : EX1(read);
6278 src2 : EX1(read);
6279 dst : EX2(write);
6280 INS01 : ISS;
6281 ALU : EX2;
6282 %}
6283
6284 // Conditional 2 operand
6285 // EG. CSEL X0, X1, X2, <cond>
6286 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6287 %{
6288 single_instruction;
6289 cr : EX1(read);
6290 src : EX1(read);
6291 dst : EX2(write);
6292 INS01 : ISS;
6293 ALU : EX2;
6294 %}
6295
6296 //------- Multiply pipeline operations --------------------
6297
6298 // Multiply reg-reg
6299 // Eg. MUL w0, w1, w2
6300 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6301 %{
6302 single_instruction;
6303 dst : WR(write);
6304 src1 : ISS(read);
6305 src2 : ISS(read);
6306 INS01 : ISS;
6307 MAC : WR;
6308 %}
6309
6310 // Multiply accumulate
6311 // Eg. MADD w0, w1, w2, w3
6312 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6313 %{
6314 single_instruction;
6315 dst : WR(write);
6316 src1 : ISS(read);
6317 src2 : ISS(read);
6318 src3 : ISS(read);
6319 INS01 : ISS;
6320 MAC : WR;
6321 %}
6322
6323 // Eg. MUL w0, w1, w2
6324 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6325 %{
6326 single_instruction;
6327 fixed_latency(3); // Maximum latency for 64 bit mul
6328 dst : WR(write);
6329 src1 : ISS(read);
6330 src2 : ISS(read);
6331 INS01 : ISS;
6332 MAC : WR;
6333 %}
6334
6335 // Multiply accumulate
6336 // Eg. MADD w0, w1, w2, w3
6337 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6338 %{
6339 single_instruction;
6340 fixed_latency(3); // Maximum latency for 64 bit mul
6341 dst : WR(write);
6342 src1 : ISS(read);
6343 src2 : ISS(read);
6344 src3 : ISS(read);
6345 INS01 : ISS;
6346 MAC : WR;
6347 %}
6348
6349 //------- Divide pipeline operations --------------------
6350
6351 // Eg. SDIV w0, w1, w2
6352 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6353 %{
6354 single_instruction;
6355 fixed_latency(8); // Maximum latency for 32 bit divide
6356 dst : WR(write);
6357 src1 : ISS(read);
6358 src2 : ISS(read);
6359 INS0 : ISS; // Can only dual issue as instruction 0
6360 DIV : WR;
6361 %}
6362
6363 // Eg. SDIV x0, x1, x2
6364 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6365 %{
6366 single_instruction;
6367 fixed_latency(16); // Maximum latency for 64 bit divide
6368 dst : WR(write);
6369 src1 : ISS(read);
6370 src2 : ISS(read);
6371 INS0 : ISS; // Can only dual issue as instruction 0
6372 DIV : WR;
6373 %}
6374
6375 //------- Load pipeline operations ------------------------
6376
6377 // Load - prefetch
6378 // Eg. PFRM <mem>
6379 pipe_class iload_prefetch(memory mem)
6380 %{
6381 single_instruction;
6382 mem : ISS(read);
6383 INS01 : ISS;
6384 LDST : WR;
6385 %}
6386
6387 // Load - reg, mem
6388 // Eg. LDR x0, <mem>
6389 pipe_class iload_reg_mem(iRegI dst, memory mem)
6390 %{
6391 single_instruction;
6392 dst : WR(write);
6393 mem : ISS(read);
6394 INS01 : ISS;
6395 LDST : WR;
6396 %}
6397
6398 // Load - reg, reg
6399 // Eg. LDR x0, [sp, x1]
6400 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6401 %{
6402 single_instruction;
6403 dst : WR(write);
6404 src : ISS(read);
6405 INS01 : ISS;
6406 LDST : WR;
6407 %}
6408
6409 //------- Store pipeline operations -----------------------
6410
6411 // Store - zr, mem
6412 // Eg. STR zr, <mem>
6413 pipe_class istore_mem(memory mem)
6414 %{
6415 single_instruction;
6416 mem : ISS(read);
6417 INS01 : ISS;
6418 LDST : WR;
6419 %}
6420
6421 // Store - reg, mem
6422 // Eg. STR x0, <mem>
6423 pipe_class istore_reg_mem(iRegI src, memory mem)
6424 %{
6425 single_instruction;
6426 mem : ISS(read);
6427 src : EX2(read);
6428 INS01 : ISS;
6429 LDST : WR;
6430 %}
6431
6432 // Store - reg, reg
6433 // Eg. STR x0, [sp, x1]
6434 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6435 %{
6436 single_instruction;
6437 dst : ISS(read);
6438 src : EX2(read);
6439 INS01 : ISS;
6440 LDST : WR;
6441 %}
6442
6443 //------- Store pipeline operations -----------------------
6444
6445 // Branch
6446 pipe_class pipe_branch()
6447 %{
6448 single_instruction;
6449 INS01 : ISS;
6450 BRANCH : EX1;
6451 %}
6452
6453 // Conditional branch
6454 pipe_class pipe_branch_cond(rFlagsReg cr)
6455 %{
6456 single_instruction;
6457 cr : EX1(read);
6458 INS01 : ISS;
6459 BRANCH : EX1;
6460 %}
6461
6462 // Compare & Branch
6463 // EG. CBZ/CBNZ
6464 pipe_class pipe_cmp_branch(iRegI op1)
6465 %{
6466 single_instruction;
6467 op1 : EX1(read);
6468 INS01 : ISS;
6469 BRANCH : EX1;
6470 %}
6471
6472 //------- Synchronisation operations ----------------------
6473
6474 // Any operation requiring serialization.
6475 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6476 pipe_class pipe_serial()
6477 %{
6478 single_instruction;
6479 force_serialization;
6480 fixed_latency(16);
6481 INS01 : ISS(2); // Cannot dual issue with any other instruction
6482 LDST : WR;
6483 %}
6484
6485 // Generic big/slow expanded idiom - also serialized
6486 pipe_class pipe_slow()
6487 %{
6488 instruction_count(10);
6489 multiple_bundles;
6490 force_serialization;
6491 fixed_latency(16);
6492 INS01 : ISS(2); // Cannot dual issue with any other instruction
6493 LDST : WR;
6494 %}
6495
6496 // Empty pipeline class
6497 pipe_class pipe_class_empty()
6498 %{
6499 single_instruction;
6500 fixed_latency(0);
6501 %}
6502
6503 // Default pipeline class.
6504 pipe_class pipe_class_default()
6505 %{
6506 single_instruction;
6507 fixed_latency(2);
6508 %}
6509
6510 // Pipeline class for compares.
6511 pipe_class pipe_class_compare()
6512 %{
6513 single_instruction;
6514 fixed_latency(16);
6515 %}
6516
6517 // Pipeline class for memory operations.
6518 pipe_class pipe_class_memory()
6519 %{
6520 single_instruction;
6521 fixed_latency(16);
6522 %}
6523
6524 // Pipeline class for call.
6525 pipe_class pipe_class_call()
6526 %{
6527 single_instruction;
6528 fixed_latency(100);
6529 %}
6530
6531 // Define the class for the Nop node.
6532 define %{
6533 MachNop = pipe_class_empty;
6534 %}
6535
6536 %}
6537 //----------INSTRUCTIONS-------------------------------------------------------
6538 //
6539 // match -- States which machine-independent subtree may be replaced
6540 // by this instruction.
6541 // ins_cost -- The estimated cost of this instruction is used by instruction
6542 // selection to identify a minimum cost tree of machine
6543 // instructions that matches a tree of machine-independent
6544 // instructions.
6545 // format -- A string providing the disassembly for this instruction.
6546 // The value of an instruction's operand may be inserted
6547 // by referring to it with a '$' prefix.
6548 // opcode -- Three instruction opcodes may be provided. These are referred
6549 // to within an encode class as $primary, $secondary, and $tertiary
6550 // rrspectively. The primary opcode is commonly used to
6551 // indicate the type of machine instruction, while secondary
6552 // and tertiary are often used for prefix options or addressing
6553 // modes.
6554 // ins_encode -- A list of encode classes with parameters. The encode class
6555 // name must have been defined in an 'enc_class' specification
6556 // in the encode section of the architecture description.
6557
6558 // ============================================================================
6559 // Memory (Load/Store) Instructions
6560
6561 // Load Instructions
6562
6563 // Load Byte (8 bit signed)
6564 instruct loadB(iRegINoSp dst, memory1 mem)
6565 %{
6566 match(Set dst (LoadB mem));
6567 predicate(!needs_acquiring_load(n));
6568
6569 ins_cost(4 * INSN_COST);
6570 format %{ "ldrsbw $dst, $mem\t# byte" %}
6571
6572 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6573
6574 ins_pipe(iload_reg_mem);
6575 %}
6576
6577 // Load Byte (8 bit signed) into long
6578 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6579 %{
6580 match(Set dst (ConvI2L (LoadB mem)));
6581 predicate(!needs_acquiring_load(n->in(1)));
6582
6583 ins_cost(4 * INSN_COST);
6584 format %{ "ldrsb $dst, $mem\t# byte" %}
6585
6586 ins_encode(aarch64_enc_ldrsb(dst, mem));
6587
6588 ins_pipe(iload_reg_mem);
6589 %}
6590
6591 // Load Byte (8 bit unsigned)
6592 instruct loadUB(iRegINoSp dst, memory1 mem)
6593 %{
6594 match(Set dst (LoadUB mem));
6595 predicate(!needs_acquiring_load(n));
6596
6597 ins_cost(4 * INSN_COST);
6598 format %{ "ldrbw $dst, $mem\t# byte" %}
6599
6600 ins_encode(aarch64_enc_ldrb(dst, mem));
6601
6602 ins_pipe(iload_reg_mem);
6603 %}
6604
6605 // Load Byte (8 bit unsigned) into long
6606 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6607 %{
6608 match(Set dst (ConvI2L (LoadUB mem)));
6609 predicate(!needs_acquiring_load(n->in(1)));
6610
6611 ins_cost(4 * INSN_COST);
6612 format %{ "ldrb $dst, $mem\t# byte" %}
6613
6614 ins_encode(aarch64_enc_ldrb(dst, mem));
6615
6616 ins_pipe(iload_reg_mem);
6617 %}
6618
6619 // Load Short (16 bit signed)
6620 instruct loadS(iRegINoSp dst, memory2 mem)
6621 %{
6622 match(Set dst (LoadS mem));
6623 predicate(!needs_acquiring_load(n));
6624
6625 ins_cost(4 * INSN_COST);
6626 format %{ "ldrshw $dst, $mem\t# short" %}
6627
6628 ins_encode(aarch64_enc_ldrshw(dst, mem));
6629
6630 ins_pipe(iload_reg_mem);
6631 %}
6632
6633 // Load Short (16 bit signed) into long
6634 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6635 %{
6636 match(Set dst (ConvI2L (LoadS mem)));
6637 predicate(!needs_acquiring_load(n->in(1)));
6638
6639 ins_cost(4 * INSN_COST);
6640 format %{ "ldrsh $dst, $mem\t# short" %}
6641
6642 ins_encode(aarch64_enc_ldrsh(dst, mem));
6643
6644 ins_pipe(iload_reg_mem);
6645 %}
6646
6647 // Load Char (16 bit unsigned)
6648 instruct loadUS(iRegINoSp dst, memory2 mem)
6649 %{
6650 match(Set dst (LoadUS mem));
6651 predicate(!needs_acquiring_load(n));
6652
6653 ins_cost(4 * INSN_COST);
6654 format %{ "ldrh $dst, $mem\t# short" %}
6655
6656 ins_encode(aarch64_enc_ldrh(dst, mem));
6657
6658 ins_pipe(iload_reg_mem);
6659 %}
6660
6661 // Load Short/Char (16 bit unsigned) into long
6662 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6663 %{
6664 match(Set dst (ConvI2L (LoadUS mem)));
6665 predicate(!needs_acquiring_load(n->in(1)));
6666
6667 ins_cost(4 * INSN_COST);
6668 format %{ "ldrh $dst, $mem\t# short" %}
6669
6670 ins_encode(aarch64_enc_ldrh(dst, mem));
6671
6672 ins_pipe(iload_reg_mem);
6673 %}
6674
6675 // Load Integer (32 bit signed)
6676 instruct loadI(iRegINoSp dst, memory4 mem)
6677 %{
6678 match(Set dst (LoadI mem));
6679 predicate(!needs_acquiring_load(n));
6680
6681 ins_cost(4 * INSN_COST);
6682 format %{ "ldrw $dst, $mem\t# int" %}
6683
6684 ins_encode(aarch64_enc_ldrw(dst, mem));
6685
6686 ins_pipe(iload_reg_mem);
6687 %}
6688
6689 // Load Integer (32 bit signed) into long
6690 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6691 %{
6692 match(Set dst (ConvI2L (LoadI mem)));
6693 predicate(!needs_acquiring_load(n->in(1)));
6694
6695 ins_cost(4 * INSN_COST);
6696 format %{ "ldrsw $dst, $mem\t# int" %}
6697
6698 ins_encode(aarch64_enc_ldrsw(dst, mem));
6699
6700 ins_pipe(iload_reg_mem);
6701 %}
6702
6703 // Load Integer (32 bit unsigned) into long
6704 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6705 %{
6706 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6707 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6708
6709 ins_cost(4 * INSN_COST);
6710 format %{ "ldrw $dst, $mem\t# int" %}
6711
6712 ins_encode(aarch64_enc_ldrw(dst, mem));
6713
6714 ins_pipe(iload_reg_mem);
6715 %}
6716
6717 // Load Long (64 bit signed)
6718 instruct loadL(iRegLNoSp dst, memory8 mem)
6719 %{
6720 match(Set dst (LoadL mem));
6721 predicate(!needs_acquiring_load(n));
6722
6723 ins_cost(4 * INSN_COST);
6724 format %{ "ldr $dst, $mem\t# int" %}
6725
6726 ins_encode(aarch64_enc_ldr(dst, mem));
6727
6728 ins_pipe(iload_reg_mem);
6729 %}
6730
6731 // Load Range
6732 instruct loadRange(iRegINoSp dst, memory4 mem)
6733 %{
6734 match(Set dst (LoadRange mem));
6735
6736 ins_cost(4 * INSN_COST);
6737 format %{ "ldrw $dst, $mem\t# range" %}
6738
6739 ins_encode(aarch64_enc_ldrw(dst, mem));
6740
6741 ins_pipe(iload_reg_mem);
6742 %}
6743
6744 // Load Pointer
6745 instruct loadP(iRegPNoSp dst, memory8 mem)
6746 %{
6747 match(Set dst (LoadP mem));
6748 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6749
6750 ins_cost(4 * INSN_COST);
6751 format %{ "ldr $dst, $mem\t# ptr" %}
6752
6753 ins_encode(aarch64_enc_ldr(dst, mem));
6754
6755 ins_pipe(iload_reg_mem);
6756 %}
6757
6758 // Load Compressed Pointer
6759 instruct loadN(iRegNNoSp dst, memory4 mem)
6760 %{
6761 match(Set dst (LoadN mem));
6762 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6763
6764 ins_cost(4 * INSN_COST);
6765 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6766
6767 ins_encode(aarch64_enc_ldrw(dst, mem));
6768
6769 ins_pipe(iload_reg_mem);
6770 %}
6771
6772 // Load Klass Pointer
6773 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6774 %{
6775 match(Set dst (LoadKlass mem));
6776 predicate(!needs_acquiring_load(n));
6777
6778 ins_cost(4 * INSN_COST);
6779 format %{ "ldr $dst, $mem\t# class" %}
6780
6781 ins_encode(aarch64_enc_ldr(dst, mem));
6782
6783 ins_pipe(iload_reg_mem);
6784 %}
6785
6786 // Load Narrow Klass Pointer
6787 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6788 %{
6789 match(Set dst (LoadNKlass mem));
6790 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6791
6792 ins_cost(4 * INSN_COST);
6793 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6794
6795 ins_encode(aarch64_enc_ldrw(dst, mem));
6796
6797 ins_pipe(iload_reg_mem);
6798 %}
6799
6800 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6801 %{
6802 match(Set dst (LoadNKlass mem));
6803 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6804
6805 ins_cost(4 * INSN_COST);
6806 format %{
6807 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6808 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6809 %}
6810 ins_encode %{
6811 // inlined aarch64_enc_ldrw
6812 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6813 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6814 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6815 %}
6816 ins_pipe(iload_reg_mem);
6817 %}
6818
6819 // Load Float
6820 instruct loadF(vRegF dst, memory4 mem)
6821 %{
6822 match(Set dst (LoadF mem));
6823 predicate(!needs_acquiring_load(n));
6824
6825 ins_cost(4 * INSN_COST);
6826 format %{ "ldrs $dst, $mem\t# float" %}
6827
6828 ins_encode( aarch64_enc_ldrs(dst, mem) );
6829
6830 ins_pipe(pipe_class_memory);
6831 %}
6832
6833 // Load Double
6834 instruct loadD(vRegD dst, memory8 mem)
6835 %{
6836 match(Set dst (LoadD mem));
6837 predicate(!needs_acquiring_load(n));
6838
6839 ins_cost(4 * INSN_COST);
6840 format %{ "ldrd $dst, $mem\t# double" %}
6841
6842 ins_encode( aarch64_enc_ldrd(dst, mem) );
6843
6844 ins_pipe(pipe_class_memory);
6845 %}
6846
6847
6848 // Load Int Constant
6849 instruct loadConI(iRegINoSp dst, immI src)
6850 %{
6851 match(Set dst src);
6852
6853 ins_cost(INSN_COST);
6854 format %{ "mov $dst, $src\t# int" %}
6855
6856 ins_encode( aarch64_enc_movw_imm(dst, src) );
6857
6858 ins_pipe(ialu_imm);
6859 %}
6860
6861 // Load Long Constant
6862 instruct loadConL(iRegLNoSp dst, immL src)
6863 %{
6864 match(Set dst src);
6865
6866 ins_cost(INSN_COST);
6867 format %{ "mov $dst, $src\t# long" %}
6868
6869 ins_encode( aarch64_enc_mov_imm(dst, src) );
6870
6871 ins_pipe(ialu_imm);
6872 %}
6873
6874 // Load Pointer Constant
6875
6876 instruct loadConP(iRegPNoSp dst, immP con)
6877 %{
6878 match(Set dst con);
6879
6880 ins_cost(INSN_COST * 4);
6881 format %{
6882 "mov $dst, $con\t# ptr\n\t"
6883 %}
6884
6885 ins_encode(aarch64_enc_mov_p(dst, con));
6886
6887 ins_pipe(ialu_imm);
6888 %}
6889
6890 // Load Null Pointer Constant
6891
6892 instruct loadConP0(iRegPNoSp dst, immP0 con)
6893 %{
6894 match(Set dst con);
6895
6896 ins_cost(INSN_COST);
6897 format %{ "mov $dst, $con\t# nullptr ptr" %}
6898
6899 ins_encode(aarch64_enc_mov_p0(dst, con));
6900
6901 ins_pipe(ialu_imm);
6902 %}
6903
6904 // Load Pointer Constant One
6905
6906 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6907 %{
6908 match(Set dst con);
6909
6910 ins_cost(INSN_COST);
6911 format %{ "mov $dst, $con\t# nullptr ptr" %}
6912
6913 ins_encode(aarch64_enc_mov_p1(dst, con));
6914
6915 ins_pipe(ialu_imm);
6916 %}
6917
6918 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6919 %{
6920 match(Set dst con);
6921
6922 ins_cost(INSN_COST);
6923 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6924
6925 ins_encode %{
6926 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6927 %}
6928
6929 ins_pipe(ialu_imm);
6930 %}
6931
6932 // Load Narrow Pointer Constant
6933
6934 instruct loadConN(iRegNNoSp dst, immN con)
6935 %{
6936 match(Set dst con);
6937
6938 ins_cost(INSN_COST * 4);
6939 format %{ "mov $dst, $con\t# compressed ptr" %}
6940
6941 ins_encode(aarch64_enc_mov_n(dst, con));
6942
6943 ins_pipe(ialu_imm);
6944 %}
6945
6946 // Load Narrow Null Pointer Constant
6947
6948 instruct loadConN0(iRegNNoSp dst, immN0 con)
6949 %{
6950 match(Set dst con);
6951
6952 ins_cost(INSN_COST);
6953 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6954
6955 ins_encode(aarch64_enc_mov_n0(dst, con));
6956
6957 ins_pipe(ialu_imm);
6958 %}
6959
6960 // Load Narrow Klass Constant
6961
6962 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6963 %{
6964 match(Set dst con);
6965
6966 ins_cost(INSN_COST);
6967 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6968
6969 ins_encode(aarch64_enc_mov_nk(dst, con));
6970
6971 ins_pipe(ialu_imm);
6972 %}
6973
6974 // Load Packed Float Constant
6975
6976 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6977 match(Set dst con);
6978 ins_cost(INSN_COST * 4);
6979 format %{ "fmovs $dst, $con"%}
6980 ins_encode %{
6981 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6982 %}
6983
6984 ins_pipe(fp_imm_s);
6985 %}
6986
6987 // Load Float Constant
6988
6989 instruct loadConF(vRegF dst, immF con) %{
6990 match(Set dst con);
6991
6992 ins_cost(INSN_COST * 4);
6993
6994 format %{
6995 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6996 %}
6997
6998 ins_encode %{
6999 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7000 %}
7001
7002 ins_pipe(fp_load_constant_s);
7003 %}
7004
7005 // Load Packed Double Constant
7006
7007 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7008 match(Set dst con);
7009 ins_cost(INSN_COST);
7010 format %{ "fmovd $dst, $con"%}
7011 ins_encode %{
7012 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7013 %}
7014
7015 ins_pipe(fp_imm_d);
7016 %}
7017
7018 // Load Double Constant
7019
7020 instruct loadConD(vRegD dst, immD con) %{
7021 match(Set dst con);
7022
7023 ins_cost(INSN_COST * 5);
7024 format %{
7025 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7026 %}
7027
7028 ins_encode %{
7029 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7030 %}
7031
7032 ins_pipe(fp_load_constant_d);
7033 %}
7034
7035 // Load Half Float Constant
7036 instruct loadConH(vRegF dst, immH con) %{
7037 match(Set dst con);
7038 format %{ "mov rscratch1, $con\n\t"
7039 "fmov $dst, rscratch1"
7040 %}
7041 ins_encode %{
7042 __ movw(rscratch1, (uint32_t)$con$$constant);
7043 __ fmovs($dst$$FloatRegister, rscratch1);
7044 %}
7045 ins_pipe(pipe_class_default);
7046 %}
7047
7048 // Store Instructions
7049
7050 // Store Byte
7051 instruct storeB(iRegIorL2I src, memory1 mem)
7052 %{
7053 match(Set mem (StoreB mem src));
7054 predicate(!needs_releasing_store(n));
7055
7056 ins_cost(INSN_COST);
7057 format %{ "strb $src, $mem\t# byte" %}
7058
7059 ins_encode(aarch64_enc_strb(src, mem));
7060
7061 ins_pipe(istore_reg_mem);
7062 %}
7063
7064
7065 instruct storeimmB0(immI0 zero, memory1 mem)
7066 %{
7067 match(Set mem (StoreB mem zero));
7068 predicate(!needs_releasing_store(n));
7069
7070 ins_cost(INSN_COST);
7071 format %{ "strb rscractch2, $mem\t# byte" %}
7072
7073 ins_encode(aarch64_enc_strb0(mem));
7074
7075 ins_pipe(istore_mem);
7076 %}
7077
7078 // Store Char/Short
7079 instruct storeC(iRegIorL2I src, memory2 mem)
7080 %{
7081 match(Set mem (StoreC mem src));
7082 predicate(!needs_releasing_store(n));
7083
7084 ins_cost(INSN_COST);
7085 format %{ "strh $src, $mem\t# short" %}
7086
7087 ins_encode(aarch64_enc_strh(src, mem));
7088
7089 ins_pipe(istore_reg_mem);
7090 %}
7091
7092 instruct storeimmC0(immI0 zero, memory2 mem)
7093 %{
7094 match(Set mem (StoreC mem zero));
7095 predicate(!needs_releasing_store(n));
7096
7097 ins_cost(INSN_COST);
7098 format %{ "strh zr, $mem\t# short" %}
7099
7100 ins_encode(aarch64_enc_strh0(mem));
7101
7102 ins_pipe(istore_mem);
7103 %}
7104
7105 // Store Integer
7106
7107 instruct storeI(iRegIorL2I src, memory4 mem)
7108 %{
7109 match(Set mem(StoreI mem src));
7110 predicate(!needs_releasing_store(n));
7111
7112 ins_cost(INSN_COST);
7113 format %{ "strw $src, $mem\t# int" %}
7114
7115 ins_encode(aarch64_enc_strw(src, mem));
7116
7117 ins_pipe(istore_reg_mem);
7118 %}
7119
7120 instruct storeimmI0(immI0 zero, memory4 mem)
7121 %{
7122 match(Set mem(StoreI mem zero));
7123 predicate(!needs_releasing_store(n));
7124
7125 ins_cost(INSN_COST);
7126 format %{ "strw zr, $mem\t# int" %}
7127
7128 ins_encode(aarch64_enc_strw0(mem));
7129
7130 ins_pipe(istore_mem);
7131 %}
7132
7133 // Store Long (64 bit signed)
7134 instruct storeL(iRegL src, memory8 mem)
7135 %{
7136 match(Set mem (StoreL mem src));
7137 predicate(!needs_releasing_store(n));
7138
7139 ins_cost(INSN_COST);
7140 format %{ "str $src, $mem\t# int" %}
7141
7142 ins_encode(aarch64_enc_str(src, mem));
7143
7144 ins_pipe(istore_reg_mem);
7145 %}
7146
7147 // Store Long (64 bit signed)
7148 instruct storeimmL0(immL0 zero, memory8 mem)
7149 %{
7150 match(Set mem (StoreL mem zero));
7151 predicate(!needs_releasing_store(n));
7152
7153 ins_cost(INSN_COST);
7154 format %{ "str zr, $mem\t# int" %}
7155
7156 ins_encode(aarch64_enc_str0(mem));
7157
7158 ins_pipe(istore_mem);
7159 %}
7160
7161 // Store Pointer
7162 instruct storeP(iRegP src, memory8 mem)
7163 %{
7164 match(Set mem (StoreP mem src));
7165 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7166
7167 ins_cost(INSN_COST);
7168 format %{ "str $src, $mem\t# ptr" %}
7169
7170 ins_encode(aarch64_enc_str(src, mem));
7171
7172 ins_pipe(istore_reg_mem);
7173 %}
7174
7175 // Store Pointer
7176 instruct storeimmP0(immP0 zero, memory8 mem)
7177 %{
7178 match(Set mem (StoreP mem zero));
7179 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7180
7181 ins_cost(INSN_COST);
7182 format %{ "str zr, $mem\t# ptr" %}
7183
7184 ins_encode(aarch64_enc_str0(mem));
7185
7186 ins_pipe(istore_mem);
7187 %}
7188
7189 // Store Compressed Pointer
7190 instruct storeN(iRegN src, memory4 mem)
7191 %{
7192 match(Set mem (StoreN mem src));
7193 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7194
7195 ins_cost(INSN_COST);
7196 format %{ "strw $src, $mem\t# compressed ptr" %}
7197
7198 ins_encode(aarch64_enc_strw(src, mem));
7199
7200 ins_pipe(istore_reg_mem);
7201 %}
7202
7203 instruct storeImmN0(immN0 zero, memory4 mem)
7204 %{
7205 match(Set mem (StoreN mem zero));
7206 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7207
7208 ins_cost(INSN_COST);
7209 format %{ "strw zr, $mem\t# compressed ptr" %}
7210
7211 ins_encode(aarch64_enc_strw0(mem));
7212
7213 ins_pipe(istore_mem);
7214 %}
7215
7216 // Store Float
7217 instruct storeF(vRegF src, memory4 mem)
7218 %{
7219 match(Set mem (StoreF mem src));
7220 predicate(!needs_releasing_store(n));
7221
7222 ins_cost(INSN_COST);
7223 format %{ "strs $src, $mem\t# float" %}
7224
7225 ins_encode( aarch64_enc_strs(src, mem) );
7226
7227 ins_pipe(pipe_class_memory);
7228 %}
7229
7230 // TODO
7231 // implement storeImmF0 and storeFImmPacked
7232
7233 // Store Double
7234 instruct storeD(vRegD src, memory8 mem)
7235 %{
7236 match(Set mem (StoreD mem src));
7237 predicate(!needs_releasing_store(n));
7238
7239 ins_cost(INSN_COST);
7240 format %{ "strd $src, $mem\t# double" %}
7241
7242 ins_encode( aarch64_enc_strd(src, mem) );
7243
7244 ins_pipe(pipe_class_memory);
7245 %}
7246
7247 // Store Compressed Klass Pointer
7248 instruct storeNKlass(iRegN src, memory4 mem)
7249 %{
7250 predicate(!needs_releasing_store(n));
7251 match(Set mem (StoreNKlass mem src));
7252
7253 ins_cost(INSN_COST);
7254 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7255
7256 ins_encode(aarch64_enc_strw(src, mem));
7257
7258 ins_pipe(istore_reg_mem);
7259 %}
7260
7261 // TODO
7262 // implement storeImmD0 and storeDImmPacked
7263
7264 // prefetch instructions
7265 // Must be safe to execute with invalid address (cannot fault).
7266
7267 instruct prefetchalloc( memory8 mem ) %{
7268 match(PrefetchAllocation mem);
7269
7270 ins_cost(INSN_COST);
7271 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7272
7273 ins_encode( aarch64_enc_prefetchw(mem) );
7274
7275 ins_pipe(iload_prefetch);
7276 %}
7277
7278 // ---------------- volatile loads and stores ----------------
7279
7280 // Load Byte (8 bit signed)
7281 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7282 %{
7283 match(Set dst (LoadB mem));
7284
7285 ins_cost(VOLATILE_REF_COST);
7286 format %{ "ldarsb $dst, $mem\t# byte" %}
7287
7288 ins_encode(aarch64_enc_ldarsb(dst, mem));
7289
7290 ins_pipe(pipe_serial);
7291 %}
7292
7293 // Load Byte (8 bit signed) into long
7294 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7295 %{
7296 match(Set dst (ConvI2L (LoadB mem)));
7297
7298 ins_cost(VOLATILE_REF_COST);
7299 format %{ "ldarsb $dst, $mem\t# byte" %}
7300
7301 ins_encode(aarch64_enc_ldarsb(dst, mem));
7302
7303 ins_pipe(pipe_serial);
7304 %}
7305
7306 // Load Byte (8 bit unsigned)
7307 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7308 %{
7309 match(Set dst (LoadUB mem));
7310
7311 ins_cost(VOLATILE_REF_COST);
7312 format %{ "ldarb $dst, $mem\t# byte" %}
7313
7314 ins_encode(aarch64_enc_ldarb(dst, mem));
7315
7316 ins_pipe(pipe_serial);
7317 %}
7318
7319 // Load Byte (8 bit unsigned) into long
7320 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7321 %{
7322 match(Set dst (ConvI2L (LoadUB mem)));
7323
7324 ins_cost(VOLATILE_REF_COST);
7325 format %{ "ldarb $dst, $mem\t# byte" %}
7326
7327 ins_encode(aarch64_enc_ldarb(dst, mem));
7328
7329 ins_pipe(pipe_serial);
7330 %}
7331
7332 // Load Short (16 bit signed)
7333 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7334 %{
7335 match(Set dst (LoadS mem));
7336
7337 ins_cost(VOLATILE_REF_COST);
7338 format %{ "ldarshw $dst, $mem\t# short" %}
7339
7340 ins_encode(aarch64_enc_ldarshw(dst, mem));
7341
7342 ins_pipe(pipe_serial);
7343 %}
7344
7345 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7346 %{
7347 match(Set dst (LoadUS mem));
7348
7349 ins_cost(VOLATILE_REF_COST);
7350 format %{ "ldarhw $dst, $mem\t# short" %}
7351
7352 ins_encode(aarch64_enc_ldarhw(dst, mem));
7353
7354 ins_pipe(pipe_serial);
7355 %}
7356
7357 // Load Short/Char (16 bit unsigned) into long
7358 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7359 %{
7360 match(Set dst (ConvI2L (LoadUS mem)));
7361
7362 ins_cost(VOLATILE_REF_COST);
7363 format %{ "ldarh $dst, $mem\t# short" %}
7364
7365 ins_encode(aarch64_enc_ldarh(dst, mem));
7366
7367 ins_pipe(pipe_serial);
7368 %}
7369
7370 // Load Short/Char (16 bit signed) into long
7371 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7372 %{
7373 match(Set dst (ConvI2L (LoadS mem)));
7374
7375 ins_cost(VOLATILE_REF_COST);
7376 format %{ "ldarh $dst, $mem\t# short" %}
7377
7378 ins_encode(aarch64_enc_ldarsh(dst, mem));
7379
7380 ins_pipe(pipe_serial);
7381 %}
7382
7383 // Load Integer (32 bit signed)
7384 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7385 %{
7386 match(Set dst (LoadI mem));
7387
7388 ins_cost(VOLATILE_REF_COST);
7389 format %{ "ldarw $dst, $mem\t# int" %}
7390
7391 ins_encode(aarch64_enc_ldarw(dst, mem));
7392
7393 ins_pipe(pipe_serial);
7394 %}
7395
7396 // Load Integer (32 bit unsigned) into long
7397 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7398 %{
7399 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7400
7401 ins_cost(VOLATILE_REF_COST);
7402 format %{ "ldarw $dst, $mem\t# int" %}
7403
7404 ins_encode(aarch64_enc_ldarw(dst, mem));
7405
7406 ins_pipe(pipe_serial);
7407 %}
7408
7409 // Load Long (64 bit signed)
7410 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7411 %{
7412 match(Set dst (LoadL mem));
7413
7414 ins_cost(VOLATILE_REF_COST);
7415 format %{ "ldar $dst, $mem\t# int" %}
7416
7417 ins_encode(aarch64_enc_ldar(dst, mem));
7418
7419 ins_pipe(pipe_serial);
7420 %}
7421
7422 // Load Pointer
7423 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7424 %{
7425 match(Set dst (LoadP mem));
7426 predicate(n->as_Load()->barrier_data() == 0);
7427
7428 ins_cost(VOLATILE_REF_COST);
7429 format %{ "ldar $dst, $mem\t# ptr" %}
7430
7431 ins_encode(aarch64_enc_ldar(dst, mem));
7432
7433 ins_pipe(pipe_serial);
7434 %}
7435
7436 // Load Compressed Pointer
7437 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7438 %{
7439 match(Set dst (LoadN mem));
7440 predicate(n->as_Load()->barrier_data() == 0);
7441
7442 ins_cost(VOLATILE_REF_COST);
7443 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7444
7445 ins_encode(aarch64_enc_ldarw(dst, mem));
7446
7447 ins_pipe(pipe_serial);
7448 %}
7449
7450 // Load Float
7451 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7452 %{
7453 match(Set dst (LoadF mem));
7454
7455 ins_cost(VOLATILE_REF_COST);
7456 format %{ "ldars $dst, $mem\t# float" %}
7457
7458 ins_encode( aarch64_enc_fldars(dst, mem) );
7459
7460 ins_pipe(pipe_serial);
7461 %}
7462
7463 // Load Double
7464 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7465 %{
7466 match(Set dst (LoadD mem));
7467
7468 ins_cost(VOLATILE_REF_COST);
7469 format %{ "ldard $dst, $mem\t# double" %}
7470
7471 ins_encode( aarch64_enc_fldard(dst, mem) );
7472
7473 ins_pipe(pipe_serial);
7474 %}
7475
7476 // Store Byte
7477 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set mem (StoreB mem src));
7480
7481 ins_cost(VOLATILE_REF_COST);
7482 format %{ "stlrb $src, $mem\t# byte" %}
7483
7484 ins_encode(aarch64_enc_stlrb(src, mem));
7485
7486 ins_pipe(pipe_class_memory);
7487 %}
7488
7489 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7490 %{
7491 match(Set mem (StoreB mem zero));
7492
7493 ins_cost(VOLATILE_REF_COST);
7494 format %{ "stlrb zr, $mem\t# byte" %}
7495
7496 ins_encode(aarch64_enc_stlrb0(mem));
7497
7498 ins_pipe(pipe_class_memory);
7499 %}
7500
7501 // Store Char/Short
7502 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7503 %{
7504 match(Set mem (StoreC mem src));
7505
7506 ins_cost(VOLATILE_REF_COST);
7507 format %{ "stlrh $src, $mem\t# short" %}
7508
7509 ins_encode(aarch64_enc_stlrh(src, mem));
7510
7511 ins_pipe(pipe_class_memory);
7512 %}
7513
7514 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7515 %{
7516 match(Set mem (StoreC mem zero));
7517
7518 ins_cost(VOLATILE_REF_COST);
7519 format %{ "stlrh zr, $mem\t# short" %}
7520
7521 ins_encode(aarch64_enc_stlrh0(mem));
7522
7523 ins_pipe(pipe_class_memory);
7524 %}
7525
7526 // Store Integer
7527
7528 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7529 %{
7530 match(Set mem(StoreI mem src));
7531
7532 ins_cost(VOLATILE_REF_COST);
7533 format %{ "stlrw $src, $mem\t# int" %}
7534
7535 ins_encode(aarch64_enc_stlrw(src, mem));
7536
7537 ins_pipe(pipe_class_memory);
7538 %}
7539
7540 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7541 %{
7542 match(Set mem(StoreI mem zero));
7543
7544 ins_cost(VOLATILE_REF_COST);
7545 format %{ "stlrw zr, $mem\t# int" %}
7546
7547 ins_encode(aarch64_enc_stlrw0(mem));
7548
7549 ins_pipe(pipe_class_memory);
7550 %}
7551
7552 // Store Long (64 bit signed)
7553 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7554 %{
7555 match(Set mem (StoreL mem src));
7556
7557 ins_cost(VOLATILE_REF_COST);
7558 format %{ "stlr $src, $mem\t# int" %}
7559
7560 ins_encode(aarch64_enc_stlr(src, mem));
7561
7562 ins_pipe(pipe_class_memory);
7563 %}
7564
7565 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7566 %{
7567 match(Set mem (StoreL mem zero));
7568
7569 ins_cost(VOLATILE_REF_COST);
7570 format %{ "stlr zr, $mem\t# int" %}
7571
7572 ins_encode(aarch64_enc_stlr0(mem));
7573
7574 ins_pipe(pipe_class_memory);
7575 %}
7576
7577 // Store Pointer
7578 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7579 %{
7580 match(Set mem (StoreP mem src));
7581 predicate(n->as_Store()->barrier_data() == 0);
7582
7583 ins_cost(VOLATILE_REF_COST);
7584 format %{ "stlr $src, $mem\t# ptr" %}
7585
7586 ins_encode(aarch64_enc_stlr(src, mem));
7587
7588 ins_pipe(pipe_class_memory);
7589 %}
7590
7591 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7592 %{
7593 match(Set mem (StoreP mem zero));
7594 predicate(n->as_Store()->barrier_data() == 0);
7595
7596 ins_cost(VOLATILE_REF_COST);
7597 format %{ "stlr zr, $mem\t# ptr" %}
7598
7599 ins_encode(aarch64_enc_stlr0(mem));
7600
7601 ins_pipe(pipe_class_memory);
7602 %}
7603
7604 // Store Compressed Pointer
7605 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7606 %{
7607 match(Set mem (StoreN mem src));
7608 predicate(n->as_Store()->barrier_data() == 0);
7609
7610 ins_cost(VOLATILE_REF_COST);
7611 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7612
7613 ins_encode(aarch64_enc_stlrw(src, mem));
7614
7615 ins_pipe(pipe_class_memory);
7616 %}
7617
7618 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7619 %{
7620 match(Set mem (StoreN mem zero));
7621 predicate(n->as_Store()->barrier_data() == 0);
7622
7623 ins_cost(VOLATILE_REF_COST);
7624 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7625
7626 ins_encode(aarch64_enc_stlrw0(mem));
7627
7628 ins_pipe(pipe_class_memory);
7629 %}
7630
7631 // Store Float
7632 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7633 %{
7634 match(Set mem (StoreF mem src));
7635
7636 ins_cost(VOLATILE_REF_COST);
7637 format %{ "stlrs $src, $mem\t# float" %}
7638
7639 ins_encode( aarch64_enc_fstlrs(src, mem) );
7640
7641 ins_pipe(pipe_class_memory);
7642 %}
7643
7644 // TODO
7645 // implement storeImmF0 and storeFImmPacked
7646
7647 // Store Double
7648 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7649 %{
7650 match(Set mem (StoreD mem src));
7651
7652 ins_cost(VOLATILE_REF_COST);
7653 format %{ "stlrd $src, $mem\t# double" %}
7654
7655 ins_encode( aarch64_enc_fstlrd(src, mem) );
7656
7657 ins_pipe(pipe_class_memory);
7658 %}
7659
7660 // ---------------- end of volatile loads and stores ----------------
7661
7662 instruct cacheWB(indirect addr)
7663 %{
7664 predicate(VM_Version::supports_data_cache_line_flush());
7665 match(CacheWB addr);
7666
7667 ins_cost(100);
7668 format %{"cache wb $addr" %}
7669 ins_encode %{
7670 assert($addr->index_position() < 0, "should be");
7671 assert($addr$$disp == 0, "should be");
7672 __ cache_wb(Address($addr$$base$$Register, 0));
7673 %}
7674 ins_pipe(pipe_slow); // XXX
7675 %}
7676
7677 instruct cacheWBPreSync()
7678 %{
7679 predicate(VM_Version::supports_data_cache_line_flush());
7680 match(CacheWBPreSync);
7681
7682 ins_cost(100);
7683 format %{"cache wb presync" %}
7684 ins_encode %{
7685 __ cache_wbsync(true);
7686 %}
7687 ins_pipe(pipe_slow); // XXX
7688 %}
7689
7690 instruct cacheWBPostSync()
7691 %{
7692 predicate(VM_Version::supports_data_cache_line_flush());
7693 match(CacheWBPostSync);
7694
7695 ins_cost(100);
7696 format %{"cache wb postsync" %}
7697 ins_encode %{
7698 __ cache_wbsync(false);
7699 %}
7700 ins_pipe(pipe_slow); // XXX
7701 %}
7702
7703 // ============================================================================
7704 // BSWAP Instructions
7705
7706 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7707 match(Set dst (ReverseBytesI src));
7708
7709 ins_cost(INSN_COST);
7710 format %{ "revw $dst, $src" %}
7711
7712 ins_encode %{
7713 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7714 %}
7715
7716 ins_pipe(ialu_reg);
7717 %}
7718
7719 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7720 match(Set dst (ReverseBytesL src));
7721
7722 ins_cost(INSN_COST);
7723 format %{ "rev $dst, $src" %}
7724
7725 ins_encode %{
7726 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7727 %}
7728
7729 ins_pipe(ialu_reg);
7730 %}
7731
7732 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7733 match(Set dst (ReverseBytesUS src));
7734
7735 ins_cost(INSN_COST);
7736 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7737
7738 ins_encode %{
7739 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7740 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7741 %}
7742
7743 ins_pipe(ialu_reg);
7744 %}
7745
7746 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7747 match(Set dst (ReverseBytesS src));
7748
7749 ins_cost(INSN_COST);
7750 format %{ "rev16w $dst, $src\n\t"
7751 "sbfmw $dst, $dst, #0, #15" %}
7752
7753 ins_encode %{
7754 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7755 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7756 %}
7757
7758 ins_pipe(ialu_reg);
7759 %}
7760
7761 // ============================================================================
7762 // Zero Count Instructions
7763
7764 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7765 match(Set dst (CountLeadingZerosI src));
7766
7767 ins_cost(INSN_COST);
7768 format %{ "clzw $dst, $src" %}
7769 ins_encode %{
7770 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7771 %}
7772
7773 ins_pipe(ialu_reg);
7774 %}
7775
7776 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7777 match(Set dst (CountLeadingZerosL src));
7778
7779 ins_cost(INSN_COST);
7780 format %{ "clz $dst, $src" %}
7781 ins_encode %{
7782 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7783 %}
7784
7785 ins_pipe(ialu_reg);
7786 %}
7787
7788 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7789 match(Set dst (CountTrailingZerosI src));
7790
7791 ins_cost(INSN_COST * 2);
7792 format %{ "rbitw $dst, $src\n\t"
7793 "clzw $dst, $dst" %}
7794 ins_encode %{
7795 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7796 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7797 %}
7798
7799 ins_pipe(ialu_reg);
7800 %}
7801
7802 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7803 match(Set dst (CountTrailingZerosL src));
7804
7805 ins_cost(INSN_COST * 2);
7806 format %{ "rbit $dst, $src\n\t"
7807 "clz $dst, $dst" %}
7808 ins_encode %{
7809 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7810 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7811 %}
7812
7813 ins_pipe(ialu_reg);
7814 %}
7815
7816 //---------- Population Count Instructions -------------------------------------
7817 //
7818
7819 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7820 match(Set dst (PopCountI src));
7821 effect(TEMP tmp);
7822 ins_cost(INSN_COST * 13);
7823
7824 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7825 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7826 "addv $tmp, $tmp\t# vector (8B)\n\t"
7827 "mov $dst, $tmp\t# vector (1D)" %}
7828 ins_encode %{
7829 __ fmovs($tmp$$FloatRegister, $src$$Register);
7830 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7831 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7832 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7833 %}
7834
7835 ins_pipe(pipe_class_default);
7836 %}
7837
7838 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7839 match(Set dst (PopCountI (LoadI mem)));
7840 effect(TEMP tmp);
7841 ins_cost(INSN_COST * 13);
7842
7843 format %{ "ldrs $tmp, $mem\n\t"
7844 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7845 "addv $tmp, $tmp\t# vector (8B)\n\t"
7846 "mov $dst, $tmp\t# vector (1D)" %}
7847 ins_encode %{
7848 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7849 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7850 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7851 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7852 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7853 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7854 %}
7855
7856 ins_pipe(pipe_class_default);
7857 %}
7858
7859 // Note: Long.bitCount(long) returns an int.
7860 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7861 match(Set dst (PopCountL src));
7862 effect(TEMP tmp);
7863 ins_cost(INSN_COST * 13);
7864
7865 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7866 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7867 "addv $tmp, $tmp\t# vector (8B)\n\t"
7868 "mov $dst, $tmp\t# vector (1D)" %}
7869 ins_encode %{
7870 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7871 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7872 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7873 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7874 %}
7875
7876 ins_pipe(pipe_class_default);
7877 %}
7878
7879 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7880 match(Set dst (PopCountL (LoadL mem)));
7881 effect(TEMP tmp);
7882 ins_cost(INSN_COST * 13);
7883
7884 format %{ "ldrd $tmp, $mem\n\t"
7885 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7886 "addv $tmp, $tmp\t# vector (8B)\n\t"
7887 "mov $dst, $tmp\t# vector (1D)" %}
7888 ins_encode %{
7889 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7890 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7891 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7892 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7893 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7894 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7895 %}
7896
7897 ins_pipe(pipe_class_default);
7898 %}
7899
7900 // ============================================================================
7901 // VerifyVectorAlignment Instruction
7902
7903 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7904 match(Set addr (VerifyVectorAlignment addr mask));
7905 effect(KILL cr);
7906 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7907 ins_encode %{
7908 Label Lskip;
7909 // check if masked bits of addr are zero
7910 __ tst($addr$$Register, $mask$$constant);
7911 __ br(Assembler::EQ, Lskip);
7912 __ stop("verify_vector_alignment found a misaligned vector memory access");
7913 __ bind(Lskip);
7914 %}
7915 ins_pipe(pipe_slow);
7916 %}
7917
7918 // ============================================================================
7919 // MemBar Instruction
7920
7921 instruct load_fence() %{
7922 match(LoadFence);
7923 ins_cost(VOLATILE_REF_COST);
7924
7925 format %{ "load_fence" %}
7926
7927 ins_encode %{
7928 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7929 %}
7930 ins_pipe(pipe_serial);
7931 %}
7932
7933 instruct unnecessary_membar_acquire() %{
7934 predicate(unnecessary_acquire(n));
7935 match(MemBarAcquire);
7936 ins_cost(0);
7937
7938 format %{ "membar_acquire (elided)" %}
7939
7940 ins_encode %{
7941 __ block_comment("membar_acquire (elided)");
7942 %}
7943
7944 ins_pipe(pipe_class_empty);
7945 %}
7946
7947 instruct membar_acquire() %{
7948 match(MemBarAcquire);
7949 ins_cost(VOLATILE_REF_COST);
7950
7951 format %{ "membar_acquire\n\t"
7952 "dmb ishld" %}
7953
7954 ins_encode %{
7955 __ block_comment("membar_acquire");
7956 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7957 %}
7958
7959 ins_pipe(pipe_serial);
7960 %}
7961
7962
7963 instruct membar_acquire_lock() %{
7964 match(MemBarAcquireLock);
7965 ins_cost(VOLATILE_REF_COST);
7966
7967 format %{ "membar_acquire_lock (elided)" %}
7968
7969 ins_encode %{
7970 __ block_comment("membar_acquire_lock (elided)");
7971 %}
7972
7973 ins_pipe(pipe_serial);
7974 %}
7975
7976 instruct store_fence() %{
7977 match(StoreFence);
7978 ins_cost(VOLATILE_REF_COST);
7979
7980 format %{ "store_fence" %}
7981
7982 ins_encode %{
7983 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7984 %}
7985 ins_pipe(pipe_serial);
7986 %}
7987
7988 instruct unnecessary_membar_release() %{
7989 predicate(unnecessary_release(n));
7990 match(MemBarRelease);
7991 ins_cost(0);
7992
7993 format %{ "membar_release (elided)" %}
7994
7995 ins_encode %{
7996 __ block_comment("membar_release (elided)");
7997 %}
7998 ins_pipe(pipe_serial);
7999 %}
8000
8001 instruct membar_release() %{
8002 match(MemBarRelease);
8003 ins_cost(VOLATILE_REF_COST);
8004
8005 format %{ "membar_release\n\t"
8006 "dmb ishst\n\tdmb ishld" %}
8007
8008 ins_encode %{
8009 __ block_comment("membar_release");
8010 // These will be merged if AlwaysMergeDMB is enabled.
8011 __ membar(Assembler::StoreStore);
8012 __ membar(Assembler::LoadStore);
8013 %}
8014 ins_pipe(pipe_serial);
8015 %}
8016
8017 instruct membar_storestore() %{
8018 match(MemBarStoreStore);
8019 match(StoreStoreFence);
8020 ins_cost(VOLATILE_REF_COST);
8021
8022 format %{ "MEMBAR-store-store" %}
8023
8024 ins_encode %{
8025 __ membar(Assembler::StoreStore);
8026 %}
8027 ins_pipe(pipe_serial);
8028 %}
8029
8030 instruct membar_release_lock() %{
8031 match(MemBarReleaseLock);
8032 ins_cost(VOLATILE_REF_COST);
8033
8034 format %{ "membar_release_lock (elided)" %}
8035
8036 ins_encode %{
8037 __ block_comment("membar_release_lock (elided)");
8038 %}
8039
8040 ins_pipe(pipe_serial);
8041 %}
8042
8043 instruct membar_storeload() %{
8044 match(MemBarStoreLoad);
8045 ins_cost(VOLATILE_REF_COST*100);
8046
8047 format %{ "MEMBAR-store-load\n\t"
8048 "dmb ish" %}
8049
8050 ins_encode %{
8051 __ block_comment("membar_storeload");
8052 __ membar(Assembler::StoreLoad);
8053 %}
8054
8055 ins_pipe(pipe_serial);
8056 %}
8057
8058 instruct unnecessary_membar_volatile() %{
8059 predicate(unnecessary_volatile(n));
8060 match(MemBarVolatile);
8061 ins_cost(0);
8062
8063 format %{ "membar_volatile (elided)" %}
8064
8065 ins_encode %{
8066 __ block_comment("membar_volatile (elided)");
8067 %}
8068
8069 ins_pipe(pipe_serial);
8070 %}
8071
8072 instruct membar_volatile() %{
8073 match(MemBarVolatile);
8074 ins_cost(VOLATILE_REF_COST*100);
8075
8076 format %{ "membar_volatile\n\t"
8077 "dmb ish"%}
8078
8079 ins_encode %{
8080 __ block_comment("membar_volatile");
8081 __ membar(Assembler::StoreLoad);
8082 %}
8083
8084 ins_pipe(pipe_serial);
8085 %}
8086
8087 instruct membar_full() %{
8088 match(MemBarFull);
8089 ins_cost(VOLATILE_REF_COST*100);
8090
8091 format %{ "membar_full\n\t"
8092 "dmb ish" %}
8093 ins_encode %{
8094 __ block_comment("membar_full");
8095 __ membar(Assembler::AnyAny);
8096 %}
8097
8098 ins_pipe(pipe_serial);
8099 %}
8100
8101 // ============================================================================
8102 // Cast/Convert Instructions
8103
8104 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8105 match(Set dst (CastX2P src));
8106
8107 ins_cost(INSN_COST);
8108 format %{ "mov $dst, $src\t# long -> ptr" %}
8109
8110 ins_encode %{
8111 if ($dst$$reg != $src$$reg) {
8112 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8113 }
8114 %}
8115
8116 ins_pipe(ialu_reg);
8117 %}
8118
8119 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8120 match(Set dst (CastP2X src));
8121
8122 ins_cost(INSN_COST);
8123 format %{ "mov $dst, $src\t# ptr -> long" %}
8124
8125 ins_encode %{
8126 if ($dst$$reg != $src$$reg) {
8127 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8128 }
8129 %}
8130
8131 ins_pipe(ialu_reg);
8132 %}
8133
8134 // Convert oop into int for vectors alignment masking
8135 instruct convP2I(iRegINoSp dst, iRegP src) %{
8136 match(Set dst (ConvL2I (CastP2X src)));
8137
8138 ins_cost(INSN_COST);
8139 format %{ "movw $dst, $src\t# ptr -> int" %}
8140 ins_encode %{
8141 __ movw($dst$$Register, $src$$Register);
8142 %}
8143
8144 ins_pipe(ialu_reg);
8145 %}
8146
8147 // Convert compressed oop into int for vectors alignment masking
8148 // in case of 32bit oops (heap < 4Gb).
8149 instruct convN2I(iRegINoSp dst, iRegN src)
8150 %{
8151 predicate(CompressedOops::shift() == 0);
8152 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8153
8154 ins_cost(INSN_COST);
8155 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8156 ins_encode %{
8157 __ movw($dst$$Register, $src$$Register);
8158 %}
8159
8160 ins_pipe(ialu_reg);
8161 %}
8162
8163
8164 // Convert oop pointer into compressed form
8165 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8166 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8167 match(Set dst (EncodeP src));
8168 effect(KILL cr);
8169 ins_cost(INSN_COST * 3);
8170 format %{ "encode_heap_oop $dst, $src" %}
8171 ins_encode %{
8172 Register s = $src$$Register;
8173 Register d = $dst$$Register;
8174 __ encode_heap_oop(d, s);
8175 %}
8176 ins_pipe(ialu_reg);
8177 %}
8178
8179 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8180 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8181 match(Set dst (EncodeP src));
8182 ins_cost(INSN_COST * 3);
8183 format %{ "encode_heap_oop_not_null $dst, $src" %}
8184 ins_encode %{
8185 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8186 %}
8187 ins_pipe(ialu_reg);
8188 %}
8189
8190 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8191 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8192 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8193 match(Set dst (DecodeN src));
8194 ins_cost(INSN_COST * 3);
8195 format %{ "decode_heap_oop $dst, $src" %}
8196 ins_encode %{
8197 Register s = $src$$Register;
8198 Register d = $dst$$Register;
8199 __ decode_heap_oop(d, s);
8200 %}
8201 ins_pipe(ialu_reg);
8202 %}
8203
8204 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8205 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8206 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8207 match(Set dst (DecodeN src));
8208 ins_cost(INSN_COST * 3);
8209 format %{ "decode_heap_oop_not_null $dst, $src" %}
8210 ins_encode %{
8211 Register s = $src$$Register;
8212 Register d = $dst$$Register;
8213 __ decode_heap_oop_not_null(d, s);
8214 %}
8215 ins_pipe(ialu_reg);
8216 %}
8217
8218 // n.b. AArch64 implementations of encode_klass_not_null and
8219 // decode_klass_not_null do not modify the flags register so, unlike
8220 // Intel, we don't kill CR as a side effect here
8221
8222 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8223 match(Set dst (EncodePKlass src));
8224
8225 ins_cost(INSN_COST * 3);
8226 format %{ "encode_klass_not_null $dst,$src" %}
8227
8228 ins_encode %{
8229 Register src_reg = as_Register($src$$reg);
8230 Register dst_reg = as_Register($dst$$reg);
8231 __ encode_klass_not_null(dst_reg, src_reg, rscratch1);
8232 %}
8233
8234 ins_pipe(ialu_reg);
8235 %}
8236
8237 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8238 match(Set dst (DecodeNKlass src));
8239
8240 ins_cost(INSN_COST * 3);
8241 format %{ "decode_klass_not_null $dst,$src" %}
8242
8243 ins_encode %{
8244 Register src_reg = as_Register($src$$reg);
8245 Register dst_reg = as_Register($dst$$reg);
8246 __ decode_klass_not_null(dst_reg, src_reg, rscratch1);
8247 %}
8248
8249 ins_pipe(ialu_reg);
8250 %}
8251
8252 instruct checkCastPP(iRegPNoSp dst)
8253 %{
8254 match(Set dst (CheckCastPP dst));
8255
8256 size(0);
8257 format %{ "# checkcastPP of $dst" %}
8258 ins_encode(/* empty encoding */);
8259 ins_pipe(pipe_class_empty);
8260 %}
8261
8262 instruct castPP(iRegPNoSp dst)
8263 %{
8264 match(Set dst (CastPP dst));
8265
8266 size(0);
8267 format %{ "# castPP of $dst" %}
8268 ins_encode(/* empty encoding */);
8269 ins_pipe(pipe_class_empty);
8270 %}
8271
8272 instruct castII(iRegI dst)
8273 %{
8274 predicate(VerifyConstraintCasts == 0);
8275 match(Set dst (CastII dst));
8276
8277 size(0);
8278 format %{ "# castII of $dst" %}
8279 ins_encode(/* empty encoding */);
8280 ins_cost(0);
8281 ins_pipe(pipe_class_empty);
8282 %}
8283
8284 instruct castII_checked(iRegI dst, rFlagsReg cr)
8285 %{
8286 predicate(VerifyConstraintCasts > 0);
8287 match(Set dst (CastII dst));
8288 effect(KILL cr);
8289
8290 format %{ "# castII_checked of $dst" %}
8291 ins_encode %{
8292 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8293 %}
8294 ins_pipe(pipe_slow);
8295 %}
8296
8297 // The unchecked and checked variants for CastII below both use iRegINoSp for src and dst
8298 // as some consumers of CastII node like ConvHF2F forbid the stack pointer as an input
8299 // (please see convHF2F_reg_reg rule which requires input to be in an iRegINoSp register).
8300 instruct castII_nosp(iRegINoSp dst)
8301 %{
8302 predicate(VerifyConstraintCasts == 0);
8303 match(Set dst (CastII dst));
8304
8305 size(0);
8306 format %{ "# castII of $dst" %}
8307 ins_encode(/* empty encoding */);
8308 ins_cost(0);
8309 ins_pipe(pipe_class_empty);
8310 %}
8311
8312 instruct castII_checked_nosp(iRegINoSp dst, rFlagsReg cr)
8313 %{
8314 predicate(VerifyConstraintCasts > 0);
8315 match(Set dst (CastII dst));
8316 effect(KILL cr);
8317
8318 format %{ "# castII_checked of $dst" %}
8319 ins_encode %{
8320 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8321 %}
8322 ins_pipe(pipe_slow);
8323 %}
8324
8325 instruct castLL(iRegL dst)
8326 %{
8327 predicate(VerifyConstraintCasts == 0);
8328 match(Set dst (CastLL dst));
8329
8330 size(0);
8331 format %{ "# castLL of $dst" %}
8332 ins_encode(/* empty encoding */);
8333 ins_cost(0);
8334 ins_pipe(pipe_class_empty);
8335 %}
8336
8337 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8338 %{
8339 predicate(VerifyConstraintCasts > 0);
8340 match(Set dst (CastLL dst));
8341 effect(KILL cr);
8342
8343 format %{ "# castLL_checked of $dst" %}
8344 ins_encode %{
8345 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8346 %}
8347 ins_pipe(pipe_slow);
8348 %}
8349
8350 instruct castHH(vRegF dst)
8351 %{
8352 match(Set dst (CastHH dst));
8353 size(0);
8354 format %{ "# castHH of $dst" %}
8355 ins_encode(/* empty encoding */);
8356 ins_cost(0);
8357 ins_pipe(pipe_class_empty);
8358 %}
8359
8360 instruct castFF(vRegF dst)
8361 %{
8362 match(Set dst (CastFF dst));
8363
8364 size(0);
8365 format %{ "# castFF of $dst" %}
8366 ins_encode(/* empty encoding */);
8367 ins_cost(0);
8368 ins_pipe(pipe_class_empty);
8369 %}
8370
8371 instruct castDD(vRegD dst)
8372 %{
8373 match(Set dst (CastDD dst));
8374
8375 size(0);
8376 format %{ "# castDD of $dst" %}
8377 ins_encode(/* empty encoding */);
8378 ins_cost(0);
8379 ins_pipe(pipe_class_empty);
8380 %}
8381
8382 instruct castVV(vReg dst)
8383 %{
8384 match(Set dst (CastVV dst));
8385
8386 size(0);
8387 format %{ "# castVV of $dst" %}
8388 ins_encode(/* empty encoding */);
8389 ins_cost(0);
8390 ins_pipe(pipe_class_empty);
8391 %}
8392
8393 instruct castVVMask(pRegGov dst)
8394 %{
8395 match(Set dst (CastVV dst));
8396
8397 size(0);
8398 format %{ "# castVV of $dst" %}
8399 ins_encode(/* empty encoding */);
8400 ins_cost(0);
8401 ins_pipe(pipe_class_empty);
8402 %}
8403
8404 // Manifest a CmpU result in an integer register.
8405 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8406 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8407 %{
8408 match(Set dst (CmpU3 src1 src2));
8409 effect(KILL flags);
8410
8411 ins_cost(INSN_COST * 3);
8412 format %{
8413 "cmpw $src1, $src2\n\t"
8414 "csetw $dst, ne\n\t"
8415 "cnegw $dst, lo\t# CmpU3(reg)"
8416 %}
8417 ins_encode %{
8418 __ cmpw($src1$$Register, $src2$$Register);
8419 __ csetw($dst$$Register, Assembler::NE);
8420 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8421 %}
8422
8423 ins_pipe(pipe_class_default);
8424 %}
8425
8426 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8427 %{
8428 match(Set dst (CmpU3 src1 src2));
8429 effect(KILL flags);
8430
8431 ins_cost(INSN_COST * 3);
8432 format %{
8433 "subsw zr, $src1, $src2\n\t"
8434 "csetw $dst, ne\n\t"
8435 "cnegw $dst, lo\t# CmpU3(imm)"
8436 %}
8437 ins_encode %{
8438 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8439 __ csetw($dst$$Register, Assembler::NE);
8440 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8441 %}
8442
8443 ins_pipe(pipe_class_default);
8444 %}
8445
8446 // Manifest a CmpUL result in an integer register.
8447 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8448 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8449 %{
8450 match(Set dst (CmpUL3 src1 src2));
8451 effect(KILL flags);
8452
8453 ins_cost(INSN_COST * 3);
8454 format %{
8455 "cmp $src1, $src2\n\t"
8456 "csetw $dst, ne\n\t"
8457 "cnegw $dst, lo\t# CmpUL3(reg)"
8458 %}
8459 ins_encode %{
8460 __ cmp($src1$$Register, $src2$$Register);
8461 __ csetw($dst$$Register, Assembler::NE);
8462 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8463 %}
8464
8465 ins_pipe(pipe_class_default);
8466 %}
8467
8468 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8469 %{
8470 match(Set dst (CmpUL3 src1 src2));
8471 effect(KILL flags);
8472
8473 ins_cost(INSN_COST * 3);
8474 format %{
8475 "subs zr, $src1, $src2\n\t"
8476 "csetw $dst, ne\n\t"
8477 "cnegw $dst, lo\t# CmpUL3(imm)"
8478 %}
8479 ins_encode %{
8480 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8481 __ csetw($dst$$Register, Assembler::NE);
8482 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8483 %}
8484
8485 ins_pipe(pipe_class_default);
8486 %}
8487
8488 // Manifest a CmpL result in an integer register.
8489 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8490 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8491 %{
8492 match(Set dst (CmpL3 src1 src2));
8493 effect(KILL flags);
8494
8495 ins_cost(INSN_COST * 3);
8496 format %{
8497 "cmp $src1, $src2\n\t"
8498 "csetw $dst, ne\n\t"
8499 "cnegw $dst, lt\t# CmpL3(reg)"
8500 %}
8501 ins_encode %{
8502 __ cmp($src1$$Register, $src2$$Register);
8503 __ csetw($dst$$Register, Assembler::NE);
8504 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8505 %}
8506
8507 ins_pipe(pipe_class_default);
8508 %}
8509
8510 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8511 %{
8512 match(Set dst (CmpL3 src1 src2));
8513 effect(KILL flags);
8514
8515 ins_cost(INSN_COST * 3);
8516 format %{
8517 "subs zr, $src1, $src2\n\t"
8518 "csetw $dst, ne\n\t"
8519 "cnegw $dst, lt\t# CmpL3(imm)"
8520 %}
8521 ins_encode %{
8522 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8523 __ csetw($dst$$Register, Assembler::NE);
8524 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8525 %}
8526
8527 ins_pipe(pipe_class_default);
8528 %}
8529
8530 // ============================================================================
8531 // Conditional Move Instructions
8532
8533 // n.b. we have identical rules for both a signed compare op (cmpOp)
8534 // and an unsigned compare op (cmpOpU). it would be nice if we could
8535 // define an op class which merged both inputs and use it to type the
8536 // argument to a single rule. unfortunatelyt his fails because the
8537 // opclass does not live up to the COND_INTER interface of its
8538 // component operands. When the generic code tries to negate the
8539 // operand it ends up running the generci Machoper::negate method
8540 // which throws a ShouldNotHappen. So, we have to provide two flavours
8541 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8542
8543 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8544 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8545
8546 ins_cost(INSN_COST * 2);
8547 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8548
8549 ins_encode %{
8550 __ cselw(as_Register($dst$$reg),
8551 as_Register($src2$$reg),
8552 as_Register($src1$$reg),
8553 (Assembler::Condition)$cmp$$cmpcode);
8554 %}
8555
8556 ins_pipe(icond_reg_reg);
8557 %}
8558
8559 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8560 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8561
8562 ins_cost(INSN_COST * 2);
8563 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8564
8565 ins_encode %{
8566 __ cselw(as_Register($dst$$reg),
8567 as_Register($src2$$reg),
8568 as_Register($src1$$reg),
8569 (Assembler::Condition)$cmp$$cmpcode);
8570 %}
8571
8572 ins_pipe(icond_reg_reg);
8573 %}
8574
8575 // special cases where one arg is zero
8576
8577 // n.b. this is selected in preference to the rule above because it
8578 // avoids loading constant 0 into a source register
8579
8580 // TODO
8581 // we ought only to be able to cull one of these variants as the ideal
8582 // transforms ought always to order the zero consistently (to left/right?)
8583
8584 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8585 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8586
8587 ins_cost(INSN_COST * 2);
8588 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8589
8590 ins_encode %{
8591 __ cselw(as_Register($dst$$reg),
8592 as_Register($src$$reg),
8593 zr,
8594 (Assembler::Condition)$cmp$$cmpcode);
8595 %}
8596
8597 ins_pipe(icond_reg);
8598 %}
8599
8600 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8601 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8602
8603 ins_cost(INSN_COST * 2);
8604 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8605
8606 ins_encode %{
8607 __ cselw(as_Register($dst$$reg),
8608 as_Register($src$$reg),
8609 zr,
8610 (Assembler::Condition)$cmp$$cmpcode);
8611 %}
8612
8613 ins_pipe(icond_reg);
8614 %}
8615
8616 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8617 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8618
8619 ins_cost(INSN_COST * 2);
8620 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8621
8622 ins_encode %{
8623 __ cselw(as_Register($dst$$reg),
8624 zr,
8625 as_Register($src$$reg),
8626 (Assembler::Condition)$cmp$$cmpcode);
8627 %}
8628
8629 ins_pipe(icond_reg);
8630 %}
8631
8632 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8633 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8634
8635 ins_cost(INSN_COST * 2);
8636 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8637
8638 ins_encode %{
8639 __ cselw(as_Register($dst$$reg),
8640 zr,
8641 as_Register($src$$reg),
8642 (Assembler::Condition)$cmp$$cmpcode);
8643 %}
8644
8645 ins_pipe(icond_reg);
8646 %}
8647
8648 // special case for creating a boolean 0 or 1
8649
8650 // n.b. this is selected in preference to the rule above because it
8651 // avoids loading constants 0 and 1 into a source register
8652
8653 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8654 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8655
8656 ins_cost(INSN_COST * 2);
8657 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8658
8659 ins_encode %{
8660 // equivalently
8661 // cset(as_Register($dst$$reg),
8662 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8663 __ csincw(as_Register($dst$$reg),
8664 zr,
8665 zr,
8666 (Assembler::Condition)$cmp$$cmpcode);
8667 %}
8668
8669 ins_pipe(icond_none);
8670 %}
8671
8672 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8673 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8674
8675 ins_cost(INSN_COST * 2);
8676 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8677
8678 ins_encode %{
8679 // equivalently
8680 // cset(as_Register($dst$$reg),
8681 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8682 __ csincw(as_Register($dst$$reg),
8683 zr,
8684 zr,
8685 (Assembler::Condition)$cmp$$cmpcode);
8686 %}
8687
8688 ins_pipe(icond_none);
8689 %}
8690
8691 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8692 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8693
8694 ins_cost(INSN_COST * 2);
8695 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8696
8697 ins_encode %{
8698 __ csel(as_Register($dst$$reg),
8699 as_Register($src2$$reg),
8700 as_Register($src1$$reg),
8701 (Assembler::Condition)$cmp$$cmpcode);
8702 %}
8703
8704 ins_pipe(icond_reg_reg);
8705 %}
8706
8707 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8708 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8709
8710 ins_cost(INSN_COST * 2);
8711 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8712
8713 ins_encode %{
8714 __ csel(as_Register($dst$$reg),
8715 as_Register($src2$$reg),
8716 as_Register($src1$$reg),
8717 (Assembler::Condition)$cmp$$cmpcode);
8718 %}
8719
8720 ins_pipe(icond_reg_reg);
8721 %}
8722
8723 // special cases where one arg is zero
8724
8725 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8726 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8727
8728 ins_cost(INSN_COST * 2);
8729 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8730
8731 ins_encode %{
8732 __ csel(as_Register($dst$$reg),
8733 zr,
8734 as_Register($src$$reg),
8735 (Assembler::Condition)$cmp$$cmpcode);
8736 %}
8737
8738 ins_pipe(icond_reg);
8739 %}
8740
8741 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8742 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8743
8744 ins_cost(INSN_COST * 2);
8745 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8746
8747 ins_encode %{
8748 __ csel(as_Register($dst$$reg),
8749 zr,
8750 as_Register($src$$reg),
8751 (Assembler::Condition)$cmp$$cmpcode);
8752 %}
8753
8754 ins_pipe(icond_reg);
8755 %}
8756
8757 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8758 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8759
8760 ins_cost(INSN_COST * 2);
8761 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8762
8763 ins_encode %{
8764 __ csel(as_Register($dst$$reg),
8765 as_Register($src$$reg),
8766 zr,
8767 (Assembler::Condition)$cmp$$cmpcode);
8768 %}
8769
8770 ins_pipe(icond_reg);
8771 %}
8772
8773 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8774 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8775
8776 ins_cost(INSN_COST * 2);
8777 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8778
8779 ins_encode %{
8780 __ csel(as_Register($dst$$reg),
8781 as_Register($src$$reg),
8782 zr,
8783 (Assembler::Condition)$cmp$$cmpcode);
8784 %}
8785
8786 ins_pipe(icond_reg);
8787 %}
8788
8789 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8790 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8791
8792 ins_cost(INSN_COST * 2);
8793 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8794
8795 ins_encode %{
8796 __ csel(as_Register($dst$$reg),
8797 as_Register($src2$$reg),
8798 as_Register($src1$$reg),
8799 (Assembler::Condition)$cmp$$cmpcode);
8800 %}
8801
8802 ins_pipe(icond_reg_reg);
8803 %}
8804
8805 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8806 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8807
8808 ins_cost(INSN_COST * 2);
8809 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8810
8811 ins_encode %{
8812 __ csel(as_Register($dst$$reg),
8813 as_Register($src2$$reg),
8814 as_Register($src1$$reg),
8815 (Assembler::Condition)$cmp$$cmpcode);
8816 %}
8817
8818 ins_pipe(icond_reg_reg);
8819 %}
8820
8821 // special cases where one arg is zero
8822
8823 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8824 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8825
8826 ins_cost(INSN_COST * 2);
8827 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8828
8829 ins_encode %{
8830 __ csel(as_Register($dst$$reg),
8831 zr,
8832 as_Register($src$$reg),
8833 (Assembler::Condition)$cmp$$cmpcode);
8834 %}
8835
8836 ins_pipe(icond_reg);
8837 %}
8838
8839 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8840 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8841
8842 ins_cost(INSN_COST * 2);
8843 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8844
8845 ins_encode %{
8846 __ csel(as_Register($dst$$reg),
8847 zr,
8848 as_Register($src$$reg),
8849 (Assembler::Condition)$cmp$$cmpcode);
8850 %}
8851
8852 ins_pipe(icond_reg);
8853 %}
8854
8855 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8856 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8857
8858 ins_cost(INSN_COST * 2);
8859 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8860
8861 ins_encode %{
8862 __ csel(as_Register($dst$$reg),
8863 as_Register($src$$reg),
8864 zr,
8865 (Assembler::Condition)$cmp$$cmpcode);
8866 %}
8867
8868 ins_pipe(icond_reg);
8869 %}
8870
8871 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8872 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8873
8874 ins_cost(INSN_COST * 2);
8875 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8876
8877 ins_encode %{
8878 __ csel(as_Register($dst$$reg),
8879 as_Register($src$$reg),
8880 zr,
8881 (Assembler::Condition)$cmp$$cmpcode);
8882 %}
8883
8884 ins_pipe(icond_reg);
8885 %}
8886
8887 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8888 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8889
8890 ins_cost(INSN_COST * 2);
8891 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8892
8893 ins_encode %{
8894 __ cselw(as_Register($dst$$reg),
8895 as_Register($src2$$reg),
8896 as_Register($src1$$reg),
8897 (Assembler::Condition)$cmp$$cmpcode);
8898 %}
8899
8900 ins_pipe(icond_reg_reg);
8901 %}
8902
8903 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8904 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8905
8906 ins_cost(INSN_COST * 2);
8907 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8908
8909 ins_encode %{
8910 __ cselw(as_Register($dst$$reg),
8911 as_Register($src2$$reg),
8912 as_Register($src1$$reg),
8913 (Assembler::Condition)$cmp$$cmpcode);
8914 %}
8915
8916 ins_pipe(icond_reg_reg);
8917 %}
8918
8919 // special cases where one arg is zero
8920
8921 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8922 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8923
8924 ins_cost(INSN_COST * 2);
8925 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8926
8927 ins_encode %{
8928 __ cselw(as_Register($dst$$reg),
8929 zr,
8930 as_Register($src$$reg),
8931 (Assembler::Condition)$cmp$$cmpcode);
8932 %}
8933
8934 ins_pipe(icond_reg);
8935 %}
8936
8937 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8938 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8939
8940 ins_cost(INSN_COST * 2);
8941 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8942
8943 ins_encode %{
8944 __ cselw(as_Register($dst$$reg),
8945 zr,
8946 as_Register($src$$reg),
8947 (Assembler::Condition)$cmp$$cmpcode);
8948 %}
8949
8950 ins_pipe(icond_reg);
8951 %}
8952
8953 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8954 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8955
8956 ins_cost(INSN_COST * 2);
8957 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8958
8959 ins_encode %{
8960 __ cselw(as_Register($dst$$reg),
8961 as_Register($src$$reg),
8962 zr,
8963 (Assembler::Condition)$cmp$$cmpcode);
8964 %}
8965
8966 ins_pipe(icond_reg);
8967 %}
8968
8969 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8970 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8971
8972 ins_cost(INSN_COST * 2);
8973 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8974
8975 ins_encode %{
8976 __ cselw(as_Register($dst$$reg),
8977 as_Register($src$$reg),
8978 zr,
8979 (Assembler::Condition)$cmp$$cmpcode);
8980 %}
8981
8982 ins_pipe(icond_reg);
8983 %}
8984
8985 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8986 %{
8987 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8988
8989 ins_cost(INSN_COST * 3);
8990
8991 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8992 ins_encode %{
8993 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8994 __ fcsels(as_FloatRegister($dst$$reg),
8995 as_FloatRegister($src2$$reg),
8996 as_FloatRegister($src1$$reg),
8997 cond);
8998 %}
8999
9000 ins_pipe(fp_cond_reg_reg_s);
9001 %}
9002
9003 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9004 %{
9005 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9006
9007 ins_cost(INSN_COST * 3);
9008
9009 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9010 ins_encode %{
9011 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9012 __ fcsels(as_FloatRegister($dst$$reg),
9013 as_FloatRegister($src2$$reg),
9014 as_FloatRegister($src1$$reg),
9015 cond);
9016 %}
9017
9018 ins_pipe(fp_cond_reg_reg_s);
9019 %}
9020
9021 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9022 %{
9023 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9024
9025 ins_cost(INSN_COST * 3);
9026
9027 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9028 ins_encode %{
9029 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9030 __ fcseld(as_FloatRegister($dst$$reg),
9031 as_FloatRegister($src2$$reg),
9032 as_FloatRegister($src1$$reg),
9033 cond);
9034 %}
9035
9036 ins_pipe(fp_cond_reg_reg_d);
9037 %}
9038
9039 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9040 %{
9041 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9042
9043 ins_cost(INSN_COST * 3);
9044
9045 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9046 ins_encode %{
9047 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9048 __ fcseld(as_FloatRegister($dst$$reg),
9049 as_FloatRegister($src2$$reg),
9050 as_FloatRegister($src1$$reg),
9051 cond);
9052 %}
9053
9054 ins_pipe(fp_cond_reg_reg_d);
9055 %}
9056
9057 // ============================================================================
9058 // Arithmetic Instructions
9059 //
9060
9061 // Integer Addition
9062
9063 // TODO
9064 // these currently employ operations which do not set CR and hence are
9065 // not flagged as killing CR but we would like to isolate the cases
9066 // where we want to set flags from those where we don't. need to work
9067 // out how to do that.
9068
9069 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9070 match(Set dst (AddI src1 src2));
9071
9072 ins_cost(INSN_COST);
9073 format %{ "addw $dst, $src1, $src2" %}
9074
9075 ins_encode %{
9076 __ addw(as_Register($dst$$reg),
9077 as_Register($src1$$reg),
9078 as_Register($src2$$reg));
9079 %}
9080
9081 ins_pipe(ialu_reg_reg);
9082 %}
9083
9084 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9085 match(Set dst (AddI src1 src2));
9086
9087 ins_cost(INSN_COST);
9088 format %{ "addw $dst, $src1, $src2" %}
9089
9090 // use opcode to indicate that this is an add not a sub
9091 opcode(0x0);
9092
9093 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9094
9095 ins_pipe(ialu_reg_imm);
9096 %}
9097
9098 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9099 match(Set dst (AddI (ConvL2I src1) src2));
9100
9101 ins_cost(INSN_COST);
9102 format %{ "addw $dst, $src1, $src2" %}
9103
9104 // use opcode to indicate that this is an add not a sub
9105 opcode(0x0);
9106
9107 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9108
9109 ins_pipe(ialu_reg_imm);
9110 %}
9111
9112 // Pointer Addition
9113 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9114 match(Set dst (AddP src1 src2));
9115
9116 ins_cost(INSN_COST);
9117 format %{ "add $dst, $src1, $src2\t# ptr" %}
9118
9119 ins_encode %{
9120 __ add(as_Register($dst$$reg),
9121 as_Register($src1$$reg),
9122 as_Register($src2$$reg));
9123 %}
9124
9125 ins_pipe(ialu_reg_reg);
9126 %}
9127
9128 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9129 match(Set dst (AddP src1 (ConvI2L src2)));
9130
9131 ins_cost(1.9 * INSN_COST);
9132 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9133
9134 ins_encode %{
9135 __ add(as_Register($dst$$reg),
9136 as_Register($src1$$reg),
9137 as_Register($src2$$reg), ext::sxtw);
9138 %}
9139
9140 ins_pipe(ialu_reg_reg);
9141 %}
9142
9143 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9144 match(Set dst (AddP src1 (LShiftL src2 scale)));
9145
9146 ins_cost(1.9 * INSN_COST);
9147 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9148
9149 ins_encode %{
9150 __ lea(as_Register($dst$$reg),
9151 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9152 Address::lsl($scale$$constant)));
9153 %}
9154
9155 ins_pipe(ialu_reg_reg_shift);
9156 %}
9157
9158 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9159 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9160
9161 ins_cost(1.9 * INSN_COST);
9162 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9163
9164 ins_encode %{
9165 __ lea(as_Register($dst$$reg),
9166 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9167 Address::sxtw($scale$$constant)));
9168 %}
9169
9170 ins_pipe(ialu_reg_reg_shift);
9171 %}
9172
9173 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9174 match(Set dst (LShiftL (ConvI2L src) scale));
9175
9176 ins_cost(INSN_COST);
9177 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9178
9179 ins_encode %{
9180 __ sbfiz(as_Register($dst$$reg),
9181 as_Register($src$$reg),
9182 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9183 %}
9184
9185 ins_pipe(ialu_reg_shift);
9186 %}
9187
9188 // Pointer Immediate Addition
9189 // n.b. this needs to be more expensive than using an indirect memory
9190 // operand
9191 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9192 match(Set dst (AddP src1 src2));
9193
9194 ins_cost(INSN_COST);
9195 format %{ "add $dst, $src1, $src2\t# ptr" %}
9196
9197 // use opcode to indicate that this is an add not a sub
9198 opcode(0x0);
9199
9200 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9201
9202 ins_pipe(ialu_reg_imm);
9203 %}
9204
9205 // Long Addition
9206 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9207
9208 match(Set dst (AddL src1 src2));
9209
9210 ins_cost(INSN_COST);
9211 format %{ "add $dst, $src1, $src2" %}
9212
9213 ins_encode %{
9214 __ add(as_Register($dst$$reg),
9215 as_Register($src1$$reg),
9216 as_Register($src2$$reg));
9217 %}
9218
9219 ins_pipe(ialu_reg_reg);
9220 %}
9221
9222 // No constant pool entries requiredLong Immediate Addition.
9223 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9224 match(Set dst (AddL src1 src2));
9225
9226 ins_cost(INSN_COST);
9227 format %{ "add $dst, $src1, $src2" %}
9228
9229 // use opcode to indicate that this is an add not a sub
9230 opcode(0x0);
9231
9232 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9233
9234 ins_pipe(ialu_reg_imm);
9235 %}
9236
9237 // Integer Subtraction
9238 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9239 match(Set dst (SubI src1 src2));
9240
9241 ins_cost(INSN_COST);
9242 format %{ "subw $dst, $src1, $src2" %}
9243
9244 ins_encode %{
9245 __ subw(as_Register($dst$$reg),
9246 as_Register($src1$$reg),
9247 as_Register($src2$$reg));
9248 %}
9249
9250 ins_pipe(ialu_reg_reg);
9251 %}
9252
9253 // Immediate Subtraction
9254 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9255 match(Set dst (SubI src1 src2));
9256
9257 ins_cost(INSN_COST);
9258 format %{ "subw $dst, $src1, $src2" %}
9259
9260 // use opcode to indicate that this is a sub not an add
9261 opcode(0x1);
9262
9263 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9264
9265 ins_pipe(ialu_reg_imm);
9266 %}
9267
9268 // Long Subtraction
9269 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9270
9271 match(Set dst (SubL src1 src2));
9272
9273 ins_cost(INSN_COST);
9274 format %{ "sub $dst, $src1, $src2" %}
9275
9276 ins_encode %{
9277 __ sub(as_Register($dst$$reg),
9278 as_Register($src1$$reg),
9279 as_Register($src2$$reg));
9280 %}
9281
9282 ins_pipe(ialu_reg_reg);
9283 %}
9284
9285 // No constant pool entries requiredLong Immediate Subtraction.
9286 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9287 match(Set dst (SubL src1 src2));
9288
9289 ins_cost(INSN_COST);
9290 format %{ "sub$dst, $src1, $src2" %}
9291
9292 // use opcode to indicate that this is a sub not an add
9293 opcode(0x1);
9294
9295 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9296
9297 ins_pipe(ialu_reg_imm);
9298 %}
9299
9300 // Integer Negation (special case for sub)
9301
9302 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9303 match(Set dst (SubI zero src));
9304
9305 ins_cost(INSN_COST);
9306 format %{ "negw $dst, $src\t# int" %}
9307
9308 ins_encode %{
9309 __ negw(as_Register($dst$$reg),
9310 as_Register($src$$reg));
9311 %}
9312
9313 ins_pipe(ialu_reg);
9314 %}
9315
9316 // Long Negation
9317
9318 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9319 match(Set dst (SubL zero src));
9320
9321 ins_cost(INSN_COST);
9322 format %{ "neg $dst, $src\t# long" %}
9323
9324 ins_encode %{
9325 __ neg(as_Register($dst$$reg),
9326 as_Register($src$$reg));
9327 %}
9328
9329 ins_pipe(ialu_reg);
9330 %}
9331
9332 // Integer Multiply
9333
9334 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9335 match(Set dst (MulI src1 src2));
9336
9337 ins_cost(INSN_COST * 3);
9338 format %{ "mulw $dst, $src1, $src2" %}
9339
9340 ins_encode %{
9341 __ mulw(as_Register($dst$$reg),
9342 as_Register($src1$$reg),
9343 as_Register($src2$$reg));
9344 %}
9345
9346 ins_pipe(imul_reg_reg);
9347 %}
9348
9349 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9350 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9351
9352 ins_cost(INSN_COST * 3);
9353 format %{ "smull $dst, $src1, $src2" %}
9354
9355 ins_encode %{
9356 __ smull(as_Register($dst$$reg),
9357 as_Register($src1$$reg),
9358 as_Register($src2$$reg));
9359 %}
9360
9361 ins_pipe(imul_reg_reg);
9362 %}
9363
9364 // Long Multiply
9365
9366 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9367 match(Set dst (MulL src1 src2));
9368
9369 ins_cost(INSN_COST * 5);
9370 format %{ "mul $dst, $src1, $src2" %}
9371
9372 ins_encode %{
9373 __ mul(as_Register($dst$$reg),
9374 as_Register($src1$$reg),
9375 as_Register($src2$$reg));
9376 %}
9377
9378 ins_pipe(lmul_reg_reg);
9379 %}
9380
9381 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9382 %{
9383 match(Set dst (MulHiL src1 src2));
9384
9385 ins_cost(INSN_COST * 7);
9386 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9387
9388 ins_encode %{
9389 __ smulh(as_Register($dst$$reg),
9390 as_Register($src1$$reg),
9391 as_Register($src2$$reg));
9392 %}
9393
9394 ins_pipe(lmul_reg_reg);
9395 %}
9396
9397 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9398 %{
9399 match(Set dst (UMulHiL src1 src2));
9400
9401 ins_cost(INSN_COST * 7);
9402 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9403
9404 ins_encode %{
9405 __ umulh(as_Register($dst$$reg),
9406 as_Register($src1$$reg),
9407 as_Register($src2$$reg));
9408 %}
9409
9410 ins_pipe(lmul_reg_reg);
9411 %}
9412
9413 // Combined Integer Multiply & Add/Sub
9414
9415 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9416 match(Set dst (AddI src3 (MulI src1 src2)));
9417
9418 ins_cost(INSN_COST * 3);
9419 format %{ "madd $dst, $src1, $src2, $src3" %}
9420
9421 ins_encode %{
9422 __ maddw(as_Register($dst$$reg),
9423 as_Register($src1$$reg),
9424 as_Register($src2$$reg),
9425 as_Register($src3$$reg));
9426 %}
9427
9428 ins_pipe(imac_reg_reg);
9429 %}
9430
9431 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9432 match(Set dst (SubI src3 (MulI src1 src2)));
9433
9434 ins_cost(INSN_COST * 3);
9435 format %{ "msub $dst, $src1, $src2, $src3" %}
9436
9437 ins_encode %{
9438 __ msubw(as_Register($dst$$reg),
9439 as_Register($src1$$reg),
9440 as_Register($src2$$reg),
9441 as_Register($src3$$reg));
9442 %}
9443
9444 ins_pipe(imac_reg_reg);
9445 %}
9446
9447 // Combined Integer Multiply & Neg
9448
9449 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9450 match(Set dst (MulI (SubI zero src1) src2));
9451
9452 ins_cost(INSN_COST * 3);
9453 format %{ "mneg $dst, $src1, $src2" %}
9454
9455 ins_encode %{
9456 __ mnegw(as_Register($dst$$reg),
9457 as_Register($src1$$reg),
9458 as_Register($src2$$reg));
9459 %}
9460
9461 ins_pipe(imac_reg_reg);
9462 %}
9463
9464 // Combined Long Multiply & Add/Sub
9465
9466 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9467 match(Set dst (AddL src3 (MulL src1 src2)));
9468
9469 ins_cost(INSN_COST * 5);
9470 format %{ "madd $dst, $src1, $src2, $src3" %}
9471
9472 ins_encode %{
9473 __ madd(as_Register($dst$$reg),
9474 as_Register($src1$$reg),
9475 as_Register($src2$$reg),
9476 as_Register($src3$$reg));
9477 %}
9478
9479 ins_pipe(lmac_reg_reg);
9480 %}
9481
9482 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9483 match(Set dst (SubL src3 (MulL src1 src2)));
9484
9485 ins_cost(INSN_COST * 5);
9486 format %{ "msub $dst, $src1, $src2, $src3" %}
9487
9488 ins_encode %{
9489 __ msub(as_Register($dst$$reg),
9490 as_Register($src1$$reg),
9491 as_Register($src2$$reg),
9492 as_Register($src3$$reg));
9493 %}
9494
9495 ins_pipe(lmac_reg_reg);
9496 %}
9497
9498 // Combined Long Multiply & Neg
9499
9500 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9501 match(Set dst (MulL (SubL zero src1) src2));
9502
9503 ins_cost(INSN_COST * 5);
9504 format %{ "mneg $dst, $src1, $src2" %}
9505
9506 ins_encode %{
9507 __ mneg(as_Register($dst$$reg),
9508 as_Register($src1$$reg),
9509 as_Register($src2$$reg));
9510 %}
9511
9512 ins_pipe(lmac_reg_reg);
9513 %}
9514
9515 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9516
9517 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9518 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9519
9520 ins_cost(INSN_COST * 3);
9521 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9522
9523 ins_encode %{
9524 __ smaddl(as_Register($dst$$reg),
9525 as_Register($src1$$reg),
9526 as_Register($src2$$reg),
9527 as_Register($src3$$reg));
9528 %}
9529
9530 ins_pipe(imac_reg_reg);
9531 %}
9532
9533 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9534 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9535
9536 ins_cost(INSN_COST * 3);
9537 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9538
9539 ins_encode %{
9540 __ smsubl(as_Register($dst$$reg),
9541 as_Register($src1$$reg),
9542 as_Register($src2$$reg),
9543 as_Register($src3$$reg));
9544 %}
9545
9546 ins_pipe(imac_reg_reg);
9547 %}
9548
9549 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9550 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9551
9552 ins_cost(INSN_COST * 3);
9553 format %{ "smnegl $dst, $src1, $src2" %}
9554
9555 ins_encode %{
9556 __ smnegl(as_Register($dst$$reg),
9557 as_Register($src1$$reg),
9558 as_Register($src2$$reg));
9559 %}
9560
9561 ins_pipe(imac_reg_reg);
9562 %}
9563
9564 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9565
9566 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9567 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9568
9569 ins_cost(INSN_COST * 5);
9570 format %{ "mulw rscratch1, $src1, $src2\n\t"
9571 "maddw $dst, $src3, $src4, rscratch1" %}
9572
9573 ins_encode %{
9574 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9575 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9576
9577 ins_pipe(imac_reg_reg);
9578 %}
9579
9580 // Integer Divide
9581
9582 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9583 match(Set dst (DivI src1 src2));
9584
9585 ins_cost(INSN_COST * 19);
9586 format %{ "sdivw $dst, $src1, $src2" %}
9587
9588 ins_encode(aarch64_enc_divw(dst, src1, src2));
9589 ins_pipe(idiv_reg_reg);
9590 %}
9591
9592 // Long Divide
9593
9594 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9595 match(Set dst (DivL src1 src2));
9596
9597 ins_cost(INSN_COST * 35);
9598 format %{ "sdiv $dst, $src1, $src2" %}
9599
9600 ins_encode(aarch64_enc_div(dst, src1, src2));
9601 ins_pipe(ldiv_reg_reg);
9602 %}
9603
9604 // Integer Remainder
9605
9606 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9607 match(Set dst (ModI src1 src2));
9608
9609 ins_cost(INSN_COST * 22);
9610 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9611 "msubw $dst, rscratch1, $src2, $src1" %}
9612
9613 ins_encode(aarch64_enc_modw(dst, src1, src2));
9614 ins_pipe(idiv_reg_reg);
9615 %}
9616
9617 // Long Remainder
9618
9619 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9620 match(Set dst (ModL src1 src2));
9621
9622 ins_cost(INSN_COST * 38);
9623 format %{ "sdiv rscratch1, $src1, $src2\n"
9624 "msub $dst, rscratch1, $src2, $src1" %}
9625
9626 ins_encode(aarch64_enc_mod(dst, src1, src2));
9627 ins_pipe(ldiv_reg_reg);
9628 %}
9629
9630 // Unsigned Integer Divide
9631
9632 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9633 match(Set dst (UDivI src1 src2));
9634
9635 ins_cost(INSN_COST * 19);
9636 format %{ "udivw $dst, $src1, $src2" %}
9637
9638 ins_encode %{
9639 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9640 %}
9641
9642 ins_pipe(idiv_reg_reg);
9643 %}
9644
9645 // Unsigned Long Divide
9646
9647 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9648 match(Set dst (UDivL src1 src2));
9649
9650 ins_cost(INSN_COST * 35);
9651 format %{ "udiv $dst, $src1, $src2" %}
9652
9653 ins_encode %{
9654 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9655 %}
9656
9657 ins_pipe(ldiv_reg_reg);
9658 %}
9659
9660 // Unsigned Integer Remainder
9661
9662 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9663 match(Set dst (UModI src1 src2));
9664
9665 ins_cost(INSN_COST * 22);
9666 format %{ "udivw rscratch1, $src1, $src2\n\t"
9667 "msubw $dst, rscratch1, $src2, $src1" %}
9668
9669 ins_encode %{
9670 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9671 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9672 %}
9673
9674 ins_pipe(idiv_reg_reg);
9675 %}
9676
9677 // Unsigned Long Remainder
9678
9679 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9680 match(Set dst (UModL src1 src2));
9681
9682 ins_cost(INSN_COST * 38);
9683 format %{ "udiv rscratch1, $src1, $src2\n"
9684 "msub $dst, rscratch1, $src2, $src1" %}
9685
9686 ins_encode %{
9687 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9688 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9689 %}
9690
9691 ins_pipe(ldiv_reg_reg);
9692 %}
9693
9694 // Integer Shifts
9695
9696 // Shift Left Register
9697 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9698 match(Set dst (LShiftI src1 src2));
9699
9700 ins_cost(INSN_COST * 2);
9701 format %{ "lslvw $dst, $src1, $src2" %}
9702
9703 ins_encode %{
9704 __ lslvw(as_Register($dst$$reg),
9705 as_Register($src1$$reg),
9706 as_Register($src2$$reg));
9707 %}
9708
9709 ins_pipe(ialu_reg_reg_vshift);
9710 %}
9711
9712 // Shift Left Immediate
9713 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9714 match(Set dst (LShiftI src1 src2));
9715
9716 ins_cost(INSN_COST);
9717 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9718
9719 ins_encode %{
9720 __ lslw(as_Register($dst$$reg),
9721 as_Register($src1$$reg),
9722 $src2$$constant & 0x1f);
9723 %}
9724
9725 ins_pipe(ialu_reg_shift);
9726 %}
9727
9728 // Shift Right Logical Register
9729 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9730 match(Set dst (URShiftI src1 src2));
9731
9732 ins_cost(INSN_COST * 2);
9733 format %{ "lsrvw $dst, $src1, $src2" %}
9734
9735 ins_encode %{
9736 __ lsrvw(as_Register($dst$$reg),
9737 as_Register($src1$$reg),
9738 as_Register($src2$$reg));
9739 %}
9740
9741 ins_pipe(ialu_reg_reg_vshift);
9742 %}
9743
9744 // Shift Right Logical Immediate
9745 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9746 match(Set dst (URShiftI src1 src2));
9747
9748 ins_cost(INSN_COST);
9749 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9750
9751 ins_encode %{
9752 __ lsrw(as_Register($dst$$reg),
9753 as_Register($src1$$reg),
9754 $src2$$constant & 0x1f);
9755 %}
9756
9757 ins_pipe(ialu_reg_shift);
9758 %}
9759
9760 // Shift Right Arithmetic Register
9761 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9762 match(Set dst (RShiftI src1 src2));
9763
9764 ins_cost(INSN_COST * 2);
9765 format %{ "asrvw $dst, $src1, $src2" %}
9766
9767 ins_encode %{
9768 __ asrvw(as_Register($dst$$reg),
9769 as_Register($src1$$reg),
9770 as_Register($src2$$reg));
9771 %}
9772
9773 ins_pipe(ialu_reg_reg_vshift);
9774 %}
9775
9776 // Shift Right Arithmetic Immediate
9777 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9778 match(Set dst (RShiftI src1 src2));
9779
9780 ins_cost(INSN_COST);
9781 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9782
9783 ins_encode %{
9784 __ asrw(as_Register($dst$$reg),
9785 as_Register($src1$$reg),
9786 $src2$$constant & 0x1f);
9787 %}
9788
9789 ins_pipe(ialu_reg_shift);
9790 %}
9791
9792 // Combined Int Mask and Right Shift (using UBFM)
9793 // TODO
9794
9795 // Long Shifts
9796
9797 // Shift Left Register
9798 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9799 match(Set dst (LShiftL src1 src2));
9800
9801 ins_cost(INSN_COST * 2);
9802 format %{ "lslv $dst, $src1, $src2" %}
9803
9804 ins_encode %{
9805 __ lslv(as_Register($dst$$reg),
9806 as_Register($src1$$reg),
9807 as_Register($src2$$reg));
9808 %}
9809
9810 ins_pipe(ialu_reg_reg_vshift);
9811 %}
9812
9813 // Shift Left Immediate
9814 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9815 match(Set dst (LShiftL src1 src2));
9816
9817 ins_cost(INSN_COST);
9818 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9819
9820 ins_encode %{
9821 __ lsl(as_Register($dst$$reg),
9822 as_Register($src1$$reg),
9823 $src2$$constant & 0x3f);
9824 %}
9825
9826 ins_pipe(ialu_reg_shift);
9827 %}
9828
9829 // Shift Right Logical Register
9830 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9831 match(Set dst (URShiftL src1 src2));
9832
9833 ins_cost(INSN_COST * 2);
9834 format %{ "lsrv $dst, $src1, $src2" %}
9835
9836 ins_encode %{
9837 __ lsrv(as_Register($dst$$reg),
9838 as_Register($src1$$reg),
9839 as_Register($src2$$reg));
9840 %}
9841
9842 ins_pipe(ialu_reg_reg_vshift);
9843 %}
9844
9845 // Shift Right Logical Immediate
9846 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9847 match(Set dst (URShiftL src1 src2));
9848
9849 ins_cost(INSN_COST);
9850 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9851
9852 ins_encode %{
9853 __ lsr(as_Register($dst$$reg),
9854 as_Register($src1$$reg),
9855 $src2$$constant & 0x3f);
9856 %}
9857
9858 ins_pipe(ialu_reg_shift);
9859 %}
9860
9861 // A special-case pattern for card table stores.
9862 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9863 match(Set dst (URShiftL (CastP2X src1) src2));
9864
9865 ins_cost(INSN_COST);
9866 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9867
9868 ins_encode %{
9869 __ lsr(as_Register($dst$$reg),
9870 as_Register($src1$$reg),
9871 $src2$$constant & 0x3f);
9872 %}
9873
9874 ins_pipe(ialu_reg_shift);
9875 %}
9876
9877 // Shift Right Arithmetic Register
9878 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9879 match(Set dst (RShiftL src1 src2));
9880
9881 ins_cost(INSN_COST * 2);
9882 format %{ "asrv $dst, $src1, $src2" %}
9883
9884 ins_encode %{
9885 __ asrv(as_Register($dst$$reg),
9886 as_Register($src1$$reg),
9887 as_Register($src2$$reg));
9888 %}
9889
9890 ins_pipe(ialu_reg_reg_vshift);
9891 %}
9892
9893 // Shift Right Arithmetic Immediate
9894 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9895 match(Set dst (RShiftL src1 src2));
9896
9897 ins_cost(INSN_COST);
9898 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9899
9900 ins_encode %{
9901 __ asr(as_Register($dst$$reg),
9902 as_Register($src1$$reg),
9903 $src2$$constant & 0x3f);
9904 %}
9905
9906 ins_pipe(ialu_reg_shift);
9907 %}
9908
9909 // BEGIN This section of the file is automatically generated. Do not edit --------------
9910 // This section is generated from aarch64_ad.m4
9911
9912 // This pattern is automatically generated from aarch64_ad.m4
9913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9914 instruct regL_not_reg(iRegLNoSp dst,
9915 iRegL src1, immL_M1 m1,
9916 rFlagsReg cr) %{
9917 match(Set dst (XorL src1 m1));
9918 ins_cost(INSN_COST);
9919 format %{ "eon $dst, $src1, zr" %}
9920
9921 ins_encode %{
9922 __ eon(as_Register($dst$$reg),
9923 as_Register($src1$$reg),
9924 zr,
9925 Assembler::LSL, 0);
9926 %}
9927
9928 ins_pipe(ialu_reg);
9929 %}
9930
9931 // This pattern is automatically generated from aarch64_ad.m4
9932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9933 instruct regI_not_reg(iRegINoSp dst,
9934 iRegIorL2I src1, immI_M1 m1,
9935 rFlagsReg cr) %{
9936 match(Set dst (XorI src1 m1));
9937 ins_cost(INSN_COST);
9938 format %{ "eonw $dst, $src1, zr" %}
9939
9940 ins_encode %{
9941 __ eonw(as_Register($dst$$reg),
9942 as_Register($src1$$reg),
9943 zr,
9944 Assembler::LSL, 0);
9945 %}
9946
9947 ins_pipe(ialu_reg);
9948 %}
9949
9950 // This pattern is automatically generated from aarch64_ad.m4
9951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9952 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9953 immI0 zero, iRegIorL2I src1, immI src2) %{
9954 match(Set dst (SubI zero (URShiftI src1 src2)));
9955
9956 ins_cost(1.9 * INSN_COST);
9957 format %{ "negw $dst, $src1, LSR $src2" %}
9958
9959 ins_encode %{
9960 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9961 Assembler::LSR, $src2$$constant & 0x1f);
9962 %}
9963
9964 ins_pipe(ialu_reg_shift);
9965 %}
9966
9967 // This pattern is automatically generated from aarch64_ad.m4
9968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9969 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9970 immI0 zero, iRegIorL2I src1, immI src2) %{
9971 match(Set dst (SubI zero (RShiftI src1 src2)));
9972
9973 ins_cost(1.9 * INSN_COST);
9974 format %{ "negw $dst, $src1, ASR $src2" %}
9975
9976 ins_encode %{
9977 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9978 Assembler::ASR, $src2$$constant & 0x1f);
9979 %}
9980
9981 ins_pipe(ialu_reg_shift);
9982 %}
9983
9984 // This pattern is automatically generated from aarch64_ad.m4
9985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9986 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9987 immI0 zero, iRegIorL2I src1, immI src2) %{
9988 match(Set dst (SubI zero (LShiftI src1 src2)));
9989
9990 ins_cost(1.9 * INSN_COST);
9991 format %{ "negw $dst, $src1, LSL $src2" %}
9992
9993 ins_encode %{
9994 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9995 Assembler::LSL, $src2$$constant & 0x1f);
9996 %}
9997
9998 ins_pipe(ialu_reg_shift);
9999 %}
10000
10001 // This pattern is automatically generated from aarch64_ad.m4
10002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10003 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10004 immL0 zero, iRegL src1, immI src2) %{
10005 match(Set dst (SubL zero (URShiftL src1 src2)));
10006
10007 ins_cost(1.9 * INSN_COST);
10008 format %{ "neg $dst, $src1, LSR $src2" %}
10009
10010 ins_encode %{
10011 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10012 Assembler::LSR, $src2$$constant & 0x3f);
10013 %}
10014
10015 ins_pipe(ialu_reg_shift);
10016 %}
10017
10018 // This pattern is automatically generated from aarch64_ad.m4
10019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10020 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10021 immL0 zero, iRegL src1, immI src2) %{
10022 match(Set dst (SubL zero (RShiftL src1 src2)));
10023
10024 ins_cost(1.9 * INSN_COST);
10025 format %{ "neg $dst, $src1, ASR $src2" %}
10026
10027 ins_encode %{
10028 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10029 Assembler::ASR, $src2$$constant & 0x3f);
10030 %}
10031
10032 ins_pipe(ialu_reg_shift);
10033 %}
10034
10035 // This pattern is automatically generated from aarch64_ad.m4
10036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10037 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10038 immL0 zero, iRegL src1, immI src2) %{
10039 match(Set dst (SubL zero (LShiftL src1 src2)));
10040
10041 ins_cost(1.9 * INSN_COST);
10042 format %{ "neg $dst, $src1, LSL $src2" %}
10043
10044 ins_encode %{
10045 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10046 Assembler::LSL, $src2$$constant & 0x3f);
10047 %}
10048
10049 ins_pipe(ialu_reg_shift);
10050 %}
10051
10052 // This pattern is automatically generated from aarch64_ad.m4
10053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10054 instruct AndI_reg_not_reg(iRegINoSp dst,
10055 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10056 match(Set dst (AndI src1 (XorI src2 m1)));
10057 ins_cost(INSN_COST);
10058 format %{ "bicw $dst, $src1, $src2" %}
10059
10060 ins_encode %{
10061 __ bicw(as_Register($dst$$reg),
10062 as_Register($src1$$reg),
10063 as_Register($src2$$reg),
10064 Assembler::LSL, 0);
10065 %}
10066
10067 ins_pipe(ialu_reg_reg);
10068 %}
10069
10070 // This pattern is automatically generated from aarch64_ad.m4
10071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10072 instruct AndL_reg_not_reg(iRegLNoSp dst,
10073 iRegL src1, iRegL src2, immL_M1 m1) %{
10074 match(Set dst (AndL src1 (XorL src2 m1)));
10075 ins_cost(INSN_COST);
10076 format %{ "bic $dst, $src1, $src2" %}
10077
10078 ins_encode %{
10079 __ bic(as_Register($dst$$reg),
10080 as_Register($src1$$reg),
10081 as_Register($src2$$reg),
10082 Assembler::LSL, 0);
10083 %}
10084
10085 ins_pipe(ialu_reg_reg);
10086 %}
10087
10088 // This pattern is automatically generated from aarch64_ad.m4
10089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10090 instruct OrI_reg_not_reg(iRegINoSp dst,
10091 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10092 match(Set dst (OrI src1 (XorI src2 m1)));
10093 ins_cost(INSN_COST);
10094 format %{ "ornw $dst, $src1, $src2" %}
10095
10096 ins_encode %{
10097 __ ornw(as_Register($dst$$reg),
10098 as_Register($src1$$reg),
10099 as_Register($src2$$reg),
10100 Assembler::LSL, 0);
10101 %}
10102
10103 ins_pipe(ialu_reg_reg);
10104 %}
10105
10106 // This pattern is automatically generated from aarch64_ad.m4
10107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10108 instruct OrL_reg_not_reg(iRegLNoSp dst,
10109 iRegL src1, iRegL src2, immL_M1 m1) %{
10110 match(Set dst (OrL src1 (XorL src2 m1)));
10111 ins_cost(INSN_COST);
10112 format %{ "orn $dst, $src1, $src2" %}
10113
10114 ins_encode %{
10115 __ orn(as_Register($dst$$reg),
10116 as_Register($src1$$reg),
10117 as_Register($src2$$reg),
10118 Assembler::LSL, 0);
10119 %}
10120
10121 ins_pipe(ialu_reg_reg);
10122 %}
10123
10124 // This pattern is automatically generated from aarch64_ad.m4
10125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10126 instruct XorI_reg_not_reg(iRegINoSp dst,
10127 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10128 match(Set dst (XorI m1 (XorI src2 src1)));
10129 ins_cost(INSN_COST);
10130 format %{ "eonw $dst, $src1, $src2" %}
10131
10132 ins_encode %{
10133 __ eonw(as_Register($dst$$reg),
10134 as_Register($src1$$reg),
10135 as_Register($src2$$reg),
10136 Assembler::LSL, 0);
10137 %}
10138
10139 ins_pipe(ialu_reg_reg);
10140 %}
10141
10142 // This pattern is automatically generated from aarch64_ad.m4
10143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10144 instruct XorL_reg_not_reg(iRegLNoSp dst,
10145 iRegL src1, iRegL src2, immL_M1 m1) %{
10146 match(Set dst (XorL m1 (XorL src2 src1)));
10147 ins_cost(INSN_COST);
10148 format %{ "eon $dst, $src1, $src2" %}
10149
10150 ins_encode %{
10151 __ eon(as_Register($dst$$reg),
10152 as_Register($src1$$reg),
10153 as_Register($src2$$reg),
10154 Assembler::LSL, 0);
10155 %}
10156
10157 ins_pipe(ialu_reg_reg);
10158 %}
10159
10160 // This pattern is automatically generated from aarch64_ad.m4
10161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10162 // val & (-1 ^ (val >>> shift)) ==> bicw
10163 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10164 iRegIorL2I src1, iRegIorL2I src2,
10165 immI src3, immI_M1 src4) %{
10166 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10167 ins_cost(1.9 * INSN_COST);
10168 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10169
10170 ins_encode %{
10171 __ bicw(as_Register($dst$$reg),
10172 as_Register($src1$$reg),
10173 as_Register($src2$$reg),
10174 Assembler::LSR,
10175 $src3$$constant & 0x1f);
10176 %}
10177
10178 ins_pipe(ialu_reg_reg_shift);
10179 %}
10180
10181 // This pattern is automatically generated from aarch64_ad.m4
10182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10183 // val & (-1 ^ (val >>> shift)) ==> bic
10184 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10185 iRegL src1, iRegL src2,
10186 immI src3, immL_M1 src4) %{
10187 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10188 ins_cost(1.9 * INSN_COST);
10189 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10190
10191 ins_encode %{
10192 __ bic(as_Register($dst$$reg),
10193 as_Register($src1$$reg),
10194 as_Register($src2$$reg),
10195 Assembler::LSR,
10196 $src3$$constant & 0x3f);
10197 %}
10198
10199 ins_pipe(ialu_reg_reg_shift);
10200 %}
10201
10202 // This pattern is automatically generated from aarch64_ad.m4
10203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10204 // val & (-1 ^ (val >> shift)) ==> bicw
10205 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10206 iRegIorL2I src1, iRegIorL2I src2,
10207 immI src3, immI_M1 src4) %{
10208 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10209 ins_cost(1.9 * INSN_COST);
10210 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10211
10212 ins_encode %{
10213 __ bicw(as_Register($dst$$reg),
10214 as_Register($src1$$reg),
10215 as_Register($src2$$reg),
10216 Assembler::ASR,
10217 $src3$$constant & 0x1f);
10218 %}
10219
10220 ins_pipe(ialu_reg_reg_shift);
10221 %}
10222
10223 // This pattern is automatically generated from aarch64_ad.m4
10224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10225 // val & (-1 ^ (val >> shift)) ==> bic
10226 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10227 iRegL src1, iRegL src2,
10228 immI src3, immL_M1 src4) %{
10229 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10230 ins_cost(1.9 * INSN_COST);
10231 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10232
10233 ins_encode %{
10234 __ bic(as_Register($dst$$reg),
10235 as_Register($src1$$reg),
10236 as_Register($src2$$reg),
10237 Assembler::ASR,
10238 $src3$$constant & 0x3f);
10239 %}
10240
10241 ins_pipe(ialu_reg_reg_shift);
10242 %}
10243
10244 // This pattern is automatically generated from aarch64_ad.m4
10245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10246 // val & (-1 ^ (val ror shift)) ==> bicw
10247 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10248 iRegIorL2I src1, iRegIorL2I src2,
10249 immI src3, immI_M1 src4) %{
10250 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10251 ins_cost(1.9 * INSN_COST);
10252 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10253
10254 ins_encode %{
10255 __ bicw(as_Register($dst$$reg),
10256 as_Register($src1$$reg),
10257 as_Register($src2$$reg),
10258 Assembler::ROR,
10259 $src3$$constant & 0x1f);
10260 %}
10261
10262 ins_pipe(ialu_reg_reg_shift);
10263 %}
10264
10265 // This pattern is automatically generated from aarch64_ad.m4
10266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10267 // val & (-1 ^ (val ror shift)) ==> bic
10268 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10269 iRegL src1, iRegL src2,
10270 immI src3, immL_M1 src4) %{
10271 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10272 ins_cost(1.9 * INSN_COST);
10273 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10274
10275 ins_encode %{
10276 __ bic(as_Register($dst$$reg),
10277 as_Register($src1$$reg),
10278 as_Register($src2$$reg),
10279 Assembler::ROR,
10280 $src3$$constant & 0x3f);
10281 %}
10282
10283 ins_pipe(ialu_reg_reg_shift);
10284 %}
10285
10286 // This pattern is automatically generated from aarch64_ad.m4
10287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10288 // val & (-1 ^ (val << shift)) ==> bicw
10289 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10290 iRegIorL2I src1, iRegIorL2I src2,
10291 immI src3, immI_M1 src4) %{
10292 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10293 ins_cost(1.9 * INSN_COST);
10294 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10295
10296 ins_encode %{
10297 __ bicw(as_Register($dst$$reg),
10298 as_Register($src1$$reg),
10299 as_Register($src2$$reg),
10300 Assembler::LSL,
10301 $src3$$constant & 0x1f);
10302 %}
10303
10304 ins_pipe(ialu_reg_reg_shift);
10305 %}
10306
10307 // This pattern is automatically generated from aarch64_ad.m4
10308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10309 // val & (-1 ^ (val << shift)) ==> bic
10310 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10311 iRegL src1, iRegL src2,
10312 immI src3, immL_M1 src4) %{
10313 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10314 ins_cost(1.9 * INSN_COST);
10315 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10316
10317 ins_encode %{
10318 __ bic(as_Register($dst$$reg),
10319 as_Register($src1$$reg),
10320 as_Register($src2$$reg),
10321 Assembler::LSL,
10322 $src3$$constant & 0x3f);
10323 %}
10324
10325 ins_pipe(ialu_reg_reg_shift);
10326 %}
10327
10328 // This pattern is automatically generated from aarch64_ad.m4
10329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10330 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10331 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10332 iRegIorL2I src1, iRegIorL2I src2,
10333 immI src3, immI_M1 src4) %{
10334 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10335 ins_cost(1.9 * INSN_COST);
10336 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10337
10338 ins_encode %{
10339 __ eonw(as_Register($dst$$reg),
10340 as_Register($src1$$reg),
10341 as_Register($src2$$reg),
10342 Assembler::LSR,
10343 $src3$$constant & 0x1f);
10344 %}
10345
10346 ins_pipe(ialu_reg_reg_shift);
10347 %}
10348
10349 // This pattern is automatically generated from aarch64_ad.m4
10350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10351 // val ^ (-1 ^ (val >>> shift)) ==> eon
10352 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10353 iRegL src1, iRegL src2,
10354 immI src3, immL_M1 src4) %{
10355 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10356 ins_cost(1.9 * INSN_COST);
10357 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10358
10359 ins_encode %{
10360 __ eon(as_Register($dst$$reg),
10361 as_Register($src1$$reg),
10362 as_Register($src2$$reg),
10363 Assembler::LSR,
10364 $src3$$constant & 0x3f);
10365 %}
10366
10367 ins_pipe(ialu_reg_reg_shift);
10368 %}
10369
10370 // This pattern is automatically generated from aarch64_ad.m4
10371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10372 // val ^ (-1 ^ (val >> shift)) ==> eonw
10373 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10374 iRegIorL2I src1, iRegIorL2I src2,
10375 immI src3, immI_M1 src4) %{
10376 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10377 ins_cost(1.9 * INSN_COST);
10378 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10379
10380 ins_encode %{
10381 __ eonw(as_Register($dst$$reg),
10382 as_Register($src1$$reg),
10383 as_Register($src2$$reg),
10384 Assembler::ASR,
10385 $src3$$constant & 0x1f);
10386 %}
10387
10388 ins_pipe(ialu_reg_reg_shift);
10389 %}
10390
10391 // This pattern is automatically generated from aarch64_ad.m4
10392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10393 // val ^ (-1 ^ (val >> shift)) ==> eon
10394 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10395 iRegL src1, iRegL src2,
10396 immI src3, immL_M1 src4) %{
10397 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10398 ins_cost(1.9 * INSN_COST);
10399 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10400
10401 ins_encode %{
10402 __ eon(as_Register($dst$$reg),
10403 as_Register($src1$$reg),
10404 as_Register($src2$$reg),
10405 Assembler::ASR,
10406 $src3$$constant & 0x3f);
10407 %}
10408
10409 ins_pipe(ialu_reg_reg_shift);
10410 %}
10411
10412 // This pattern is automatically generated from aarch64_ad.m4
10413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10414 // val ^ (-1 ^ (val ror shift)) ==> eonw
10415 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10416 iRegIorL2I src1, iRegIorL2I src2,
10417 immI src3, immI_M1 src4) %{
10418 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10419 ins_cost(1.9 * INSN_COST);
10420 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10421
10422 ins_encode %{
10423 __ eonw(as_Register($dst$$reg),
10424 as_Register($src1$$reg),
10425 as_Register($src2$$reg),
10426 Assembler::ROR,
10427 $src3$$constant & 0x1f);
10428 %}
10429
10430 ins_pipe(ialu_reg_reg_shift);
10431 %}
10432
10433 // This pattern is automatically generated from aarch64_ad.m4
10434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10435 // val ^ (-1 ^ (val ror shift)) ==> eon
10436 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10437 iRegL src1, iRegL src2,
10438 immI src3, immL_M1 src4) %{
10439 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10440 ins_cost(1.9 * INSN_COST);
10441 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10442
10443 ins_encode %{
10444 __ eon(as_Register($dst$$reg),
10445 as_Register($src1$$reg),
10446 as_Register($src2$$reg),
10447 Assembler::ROR,
10448 $src3$$constant & 0x3f);
10449 %}
10450
10451 ins_pipe(ialu_reg_reg_shift);
10452 %}
10453
10454 // This pattern is automatically generated from aarch64_ad.m4
10455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10456 // val ^ (-1 ^ (val << shift)) ==> eonw
10457 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10458 iRegIorL2I src1, iRegIorL2I src2,
10459 immI src3, immI_M1 src4) %{
10460 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10461 ins_cost(1.9 * INSN_COST);
10462 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10463
10464 ins_encode %{
10465 __ eonw(as_Register($dst$$reg),
10466 as_Register($src1$$reg),
10467 as_Register($src2$$reg),
10468 Assembler::LSL,
10469 $src3$$constant & 0x1f);
10470 %}
10471
10472 ins_pipe(ialu_reg_reg_shift);
10473 %}
10474
10475 // This pattern is automatically generated from aarch64_ad.m4
10476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10477 // val ^ (-1 ^ (val << shift)) ==> eon
10478 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10479 iRegL src1, iRegL src2,
10480 immI src3, immL_M1 src4) %{
10481 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10482 ins_cost(1.9 * INSN_COST);
10483 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10484
10485 ins_encode %{
10486 __ eon(as_Register($dst$$reg),
10487 as_Register($src1$$reg),
10488 as_Register($src2$$reg),
10489 Assembler::LSL,
10490 $src3$$constant & 0x3f);
10491 %}
10492
10493 ins_pipe(ialu_reg_reg_shift);
10494 %}
10495
10496 // This pattern is automatically generated from aarch64_ad.m4
10497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10498 // val | (-1 ^ (val >>> shift)) ==> ornw
10499 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10500 iRegIorL2I src1, iRegIorL2I src2,
10501 immI src3, immI_M1 src4) %{
10502 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10503 ins_cost(1.9 * INSN_COST);
10504 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10505
10506 ins_encode %{
10507 __ ornw(as_Register($dst$$reg),
10508 as_Register($src1$$reg),
10509 as_Register($src2$$reg),
10510 Assembler::LSR,
10511 $src3$$constant & 0x1f);
10512 %}
10513
10514 ins_pipe(ialu_reg_reg_shift);
10515 %}
10516
10517 // This pattern is automatically generated from aarch64_ad.m4
10518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10519 // val | (-1 ^ (val >>> shift)) ==> orn
10520 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10521 iRegL src1, iRegL src2,
10522 immI src3, immL_M1 src4) %{
10523 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10524 ins_cost(1.9 * INSN_COST);
10525 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10526
10527 ins_encode %{
10528 __ orn(as_Register($dst$$reg),
10529 as_Register($src1$$reg),
10530 as_Register($src2$$reg),
10531 Assembler::LSR,
10532 $src3$$constant & 0x3f);
10533 %}
10534
10535 ins_pipe(ialu_reg_reg_shift);
10536 %}
10537
10538 // This pattern is automatically generated from aarch64_ad.m4
10539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10540 // val | (-1 ^ (val >> shift)) ==> ornw
10541 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10542 iRegIorL2I src1, iRegIorL2I src2,
10543 immI src3, immI_M1 src4) %{
10544 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10545 ins_cost(1.9 * INSN_COST);
10546 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10547
10548 ins_encode %{
10549 __ ornw(as_Register($dst$$reg),
10550 as_Register($src1$$reg),
10551 as_Register($src2$$reg),
10552 Assembler::ASR,
10553 $src3$$constant & 0x1f);
10554 %}
10555
10556 ins_pipe(ialu_reg_reg_shift);
10557 %}
10558
10559 // This pattern is automatically generated from aarch64_ad.m4
10560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10561 // val | (-1 ^ (val >> shift)) ==> orn
10562 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10563 iRegL src1, iRegL src2,
10564 immI src3, immL_M1 src4) %{
10565 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10566 ins_cost(1.9 * INSN_COST);
10567 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10568
10569 ins_encode %{
10570 __ orn(as_Register($dst$$reg),
10571 as_Register($src1$$reg),
10572 as_Register($src2$$reg),
10573 Assembler::ASR,
10574 $src3$$constant & 0x3f);
10575 %}
10576
10577 ins_pipe(ialu_reg_reg_shift);
10578 %}
10579
10580 // This pattern is automatically generated from aarch64_ad.m4
10581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10582 // val | (-1 ^ (val ror shift)) ==> ornw
10583 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10584 iRegIorL2I src1, iRegIorL2I src2,
10585 immI src3, immI_M1 src4) %{
10586 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10587 ins_cost(1.9 * INSN_COST);
10588 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10589
10590 ins_encode %{
10591 __ ornw(as_Register($dst$$reg),
10592 as_Register($src1$$reg),
10593 as_Register($src2$$reg),
10594 Assembler::ROR,
10595 $src3$$constant & 0x1f);
10596 %}
10597
10598 ins_pipe(ialu_reg_reg_shift);
10599 %}
10600
10601 // This pattern is automatically generated from aarch64_ad.m4
10602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10603 // val | (-1 ^ (val ror shift)) ==> orn
10604 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10605 iRegL src1, iRegL src2,
10606 immI src3, immL_M1 src4) %{
10607 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10608 ins_cost(1.9 * INSN_COST);
10609 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10610
10611 ins_encode %{
10612 __ orn(as_Register($dst$$reg),
10613 as_Register($src1$$reg),
10614 as_Register($src2$$reg),
10615 Assembler::ROR,
10616 $src3$$constant & 0x3f);
10617 %}
10618
10619 ins_pipe(ialu_reg_reg_shift);
10620 %}
10621
10622 // This pattern is automatically generated from aarch64_ad.m4
10623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10624 // val | (-1 ^ (val << shift)) ==> ornw
10625 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10626 iRegIorL2I src1, iRegIorL2I src2,
10627 immI src3, immI_M1 src4) %{
10628 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10629 ins_cost(1.9 * INSN_COST);
10630 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10631
10632 ins_encode %{
10633 __ ornw(as_Register($dst$$reg),
10634 as_Register($src1$$reg),
10635 as_Register($src2$$reg),
10636 Assembler::LSL,
10637 $src3$$constant & 0x1f);
10638 %}
10639
10640 ins_pipe(ialu_reg_reg_shift);
10641 %}
10642
10643 // This pattern is automatically generated from aarch64_ad.m4
10644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10645 // val | (-1 ^ (val << shift)) ==> orn
10646 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10647 iRegL src1, iRegL src2,
10648 immI src3, immL_M1 src4) %{
10649 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10650 ins_cost(1.9 * INSN_COST);
10651 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10652
10653 ins_encode %{
10654 __ orn(as_Register($dst$$reg),
10655 as_Register($src1$$reg),
10656 as_Register($src2$$reg),
10657 Assembler::LSL,
10658 $src3$$constant & 0x3f);
10659 %}
10660
10661 ins_pipe(ialu_reg_reg_shift);
10662 %}
10663
10664 // This pattern is automatically generated from aarch64_ad.m4
10665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10666 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10667 iRegIorL2I src1, iRegIorL2I src2,
10668 immI src3) %{
10669 match(Set dst (AndI src1 (URShiftI src2 src3)));
10670
10671 ins_cost(1.9 * INSN_COST);
10672 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10673
10674 ins_encode %{
10675 __ andw(as_Register($dst$$reg),
10676 as_Register($src1$$reg),
10677 as_Register($src2$$reg),
10678 Assembler::LSR,
10679 $src3$$constant & 0x1f);
10680 %}
10681
10682 ins_pipe(ialu_reg_reg_shift);
10683 %}
10684
10685 // This pattern is automatically generated from aarch64_ad.m4
10686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10687 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10688 iRegL src1, iRegL src2,
10689 immI src3) %{
10690 match(Set dst (AndL src1 (URShiftL src2 src3)));
10691
10692 ins_cost(1.9 * INSN_COST);
10693 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10694
10695 ins_encode %{
10696 __ andr(as_Register($dst$$reg),
10697 as_Register($src1$$reg),
10698 as_Register($src2$$reg),
10699 Assembler::LSR,
10700 $src3$$constant & 0x3f);
10701 %}
10702
10703 ins_pipe(ialu_reg_reg_shift);
10704 %}
10705
10706 // This pattern is automatically generated from aarch64_ad.m4
10707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10708 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10709 iRegIorL2I src1, iRegIorL2I src2,
10710 immI src3) %{
10711 match(Set dst (AndI src1 (RShiftI src2 src3)));
10712
10713 ins_cost(1.9 * INSN_COST);
10714 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10715
10716 ins_encode %{
10717 __ andw(as_Register($dst$$reg),
10718 as_Register($src1$$reg),
10719 as_Register($src2$$reg),
10720 Assembler::ASR,
10721 $src3$$constant & 0x1f);
10722 %}
10723
10724 ins_pipe(ialu_reg_reg_shift);
10725 %}
10726
10727 // This pattern is automatically generated from aarch64_ad.m4
10728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10729 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10730 iRegL src1, iRegL src2,
10731 immI src3) %{
10732 match(Set dst (AndL src1 (RShiftL src2 src3)));
10733
10734 ins_cost(1.9 * INSN_COST);
10735 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10736
10737 ins_encode %{
10738 __ andr(as_Register($dst$$reg),
10739 as_Register($src1$$reg),
10740 as_Register($src2$$reg),
10741 Assembler::ASR,
10742 $src3$$constant & 0x3f);
10743 %}
10744
10745 ins_pipe(ialu_reg_reg_shift);
10746 %}
10747
10748 // This pattern is automatically generated from aarch64_ad.m4
10749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10750 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10751 iRegIorL2I src1, iRegIorL2I src2,
10752 immI src3) %{
10753 match(Set dst (AndI src1 (LShiftI src2 src3)));
10754
10755 ins_cost(1.9 * INSN_COST);
10756 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10757
10758 ins_encode %{
10759 __ andw(as_Register($dst$$reg),
10760 as_Register($src1$$reg),
10761 as_Register($src2$$reg),
10762 Assembler::LSL,
10763 $src3$$constant & 0x1f);
10764 %}
10765
10766 ins_pipe(ialu_reg_reg_shift);
10767 %}
10768
10769 // This pattern is automatically generated from aarch64_ad.m4
10770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10771 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10772 iRegL src1, iRegL src2,
10773 immI src3) %{
10774 match(Set dst (AndL src1 (LShiftL src2 src3)));
10775
10776 ins_cost(1.9 * INSN_COST);
10777 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10778
10779 ins_encode %{
10780 __ andr(as_Register($dst$$reg),
10781 as_Register($src1$$reg),
10782 as_Register($src2$$reg),
10783 Assembler::LSL,
10784 $src3$$constant & 0x3f);
10785 %}
10786
10787 ins_pipe(ialu_reg_reg_shift);
10788 %}
10789
10790 // This pattern is automatically generated from aarch64_ad.m4
10791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10792 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10793 iRegIorL2I src1, iRegIorL2I src2,
10794 immI src3) %{
10795 match(Set dst (AndI src1 (RotateRight src2 src3)));
10796
10797 ins_cost(1.9 * INSN_COST);
10798 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10799
10800 ins_encode %{
10801 __ andw(as_Register($dst$$reg),
10802 as_Register($src1$$reg),
10803 as_Register($src2$$reg),
10804 Assembler::ROR,
10805 $src3$$constant & 0x1f);
10806 %}
10807
10808 ins_pipe(ialu_reg_reg_shift);
10809 %}
10810
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10814 iRegL src1, iRegL src2,
10815 immI src3) %{
10816 match(Set dst (AndL src1 (RotateRight src2 src3)));
10817
10818 ins_cost(1.9 * INSN_COST);
10819 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10820
10821 ins_encode %{
10822 __ andr(as_Register($dst$$reg),
10823 as_Register($src1$$reg),
10824 as_Register($src2$$reg),
10825 Assembler::ROR,
10826 $src3$$constant & 0x3f);
10827 %}
10828
10829 ins_pipe(ialu_reg_reg_shift);
10830 %}
10831
10832 // This pattern is automatically generated from aarch64_ad.m4
10833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10834 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10835 iRegIorL2I src1, iRegIorL2I src2,
10836 immI src3) %{
10837 match(Set dst (XorI src1 (URShiftI src2 src3)));
10838
10839 ins_cost(1.9 * INSN_COST);
10840 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10841
10842 ins_encode %{
10843 __ eorw(as_Register($dst$$reg),
10844 as_Register($src1$$reg),
10845 as_Register($src2$$reg),
10846 Assembler::LSR,
10847 $src3$$constant & 0x1f);
10848 %}
10849
10850 ins_pipe(ialu_reg_reg_shift);
10851 %}
10852
10853 // This pattern is automatically generated from aarch64_ad.m4
10854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10855 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10856 iRegL src1, iRegL src2,
10857 immI src3) %{
10858 match(Set dst (XorL src1 (URShiftL src2 src3)));
10859
10860 ins_cost(1.9 * INSN_COST);
10861 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10862
10863 ins_encode %{
10864 __ eor(as_Register($dst$$reg),
10865 as_Register($src1$$reg),
10866 as_Register($src2$$reg),
10867 Assembler::LSR,
10868 $src3$$constant & 0x3f);
10869 %}
10870
10871 ins_pipe(ialu_reg_reg_shift);
10872 %}
10873
10874 // This pattern is automatically generated from aarch64_ad.m4
10875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10876 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10877 iRegIorL2I src1, iRegIorL2I src2,
10878 immI src3) %{
10879 match(Set dst (XorI src1 (RShiftI src2 src3)));
10880
10881 ins_cost(1.9 * INSN_COST);
10882 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10883
10884 ins_encode %{
10885 __ eorw(as_Register($dst$$reg),
10886 as_Register($src1$$reg),
10887 as_Register($src2$$reg),
10888 Assembler::ASR,
10889 $src3$$constant & 0x1f);
10890 %}
10891
10892 ins_pipe(ialu_reg_reg_shift);
10893 %}
10894
10895 // This pattern is automatically generated from aarch64_ad.m4
10896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10897 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10898 iRegL src1, iRegL src2,
10899 immI src3) %{
10900 match(Set dst (XorL src1 (RShiftL src2 src3)));
10901
10902 ins_cost(1.9 * INSN_COST);
10903 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10904
10905 ins_encode %{
10906 __ eor(as_Register($dst$$reg),
10907 as_Register($src1$$reg),
10908 as_Register($src2$$reg),
10909 Assembler::ASR,
10910 $src3$$constant & 0x3f);
10911 %}
10912
10913 ins_pipe(ialu_reg_reg_shift);
10914 %}
10915
10916 // This pattern is automatically generated from aarch64_ad.m4
10917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10918 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10919 iRegIorL2I src1, iRegIorL2I src2,
10920 immI src3) %{
10921 match(Set dst (XorI src1 (LShiftI src2 src3)));
10922
10923 ins_cost(1.9 * INSN_COST);
10924 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10925
10926 ins_encode %{
10927 __ eorw(as_Register($dst$$reg),
10928 as_Register($src1$$reg),
10929 as_Register($src2$$reg),
10930 Assembler::LSL,
10931 $src3$$constant & 0x1f);
10932 %}
10933
10934 ins_pipe(ialu_reg_reg_shift);
10935 %}
10936
10937 // This pattern is automatically generated from aarch64_ad.m4
10938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10939 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10940 iRegL src1, iRegL src2,
10941 immI src3) %{
10942 match(Set dst (XorL src1 (LShiftL src2 src3)));
10943
10944 ins_cost(1.9 * INSN_COST);
10945 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10946
10947 ins_encode %{
10948 __ eor(as_Register($dst$$reg),
10949 as_Register($src1$$reg),
10950 as_Register($src2$$reg),
10951 Assembler::LSL,
10952 $src3$$constant & 0x3f);
10953 %}
10954
10955 ins_pipe(ialu_reg_reg_shift);
10956 %}
10957
10958 // This pattern is automatically generated from aarch64_ad.m4
10959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10960 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10961 iRegIorL2I src1, iRegIorL2I src2,
10962 immI src3) %{
10963 match(Set dst (XorI src1 (RotateRight src2 src3)));
10964
10965 ins_cost(1.9 * INSN_COST);
10966 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10967
10968 ins_encode %{
10969 __ eorw(as_Register($dst$$reg),
10970 as_Register($src1$$reg),
10971 as_Register($src2$$reg),
10972 Assembler::ROR,
10973 $src3$$constant & 0x1f);
10974 %}
10975
10976 ins_pipe(ialu_reg_reg_shift);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10982 iRegL src1, iRegL src2,
10983 immI src3) %{
10984 match(Set dst (XorL src1 (RotateRight src2 src3)));
10985
10986 ins_cost(1.9 * INSN_COST);
10987 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10988
10989 ins_encode %{
10990 __ eor(as_Register($dst$$reg),
10991 as_Register($src1$$reg),
10992 as_Register($src2$$reg),
10993 Assembler::ROR,
10994 $src3$$constant & 0x3f);
10995 %}
10996
10997 ins_pipe(ialu_reg_reg_shift);
10998 %}
10999
11000 // This pattern is automatically generated from aarch64_ad.m4
11001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11002 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11003 iRegIorL2I src1, iRegIorL2I src2,
11004 immI src3) %{
11005 match(Set dst (OrI src1 (URShiftI src2 src3)));
11006
11007 ins_cost(1.9 * INSN_COST);
11008 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11009
11010 ins_encode %{
11011 __ orrw(as_Register($dst$$reg),
11012 as_Register($src1$$reg),
11013 as_Register($src2$$reg),
11014 Assembler::LSR,
11015 $src3$$constant & 0x1f);
11016 %}
11017
11018 ins_pipe(ialu_reg_reg_shift);
11019 %}
11020
11021 // This pattern is automatically generated from aarch64_ad.m4
11022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11023 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11024 iRegL src1, iRegL src2,
11025 immI src3) %{
11026 match(Set dst (OrL src1 (URShiftL src2 src3)));
11027
11028 ins_cost(1.9 * INSN_COST);
11029 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11030
11031 ins_encode %{
11032 __ orr(as_Register($dst$$reg),
11033 as_Register($src1$$reg),
11034 as_Register($src2$$reg),
11035 Assembler::LSR,
11036 $src3$$constant & 0x3f);
11037 %}
11038
11039 ins_pipe(ialu_reg_reg_shift);
11040 %}
11041
11042 // This pattern is automatically generated from aarch64_ad.m4
11043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11044 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11045 iRegIorL2I src1, iRegIorL2I src2,
11046 immI src3) %{
11047 match(Set dst (OrI src1 (RShiftI src2 src3)));
11048
11049 ins_cost(1.9 * INSN_COST);
11050 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11051
11052 ins_encode %{
11053 __ orrw(as_Register($dst$$reg),
11054 as_Register($src1$$reg),
11055 as_Register($src2$$reg),
11056 Assembler::ASR,
11057 $src3$$constant & 0x1f);
11058 %}
11059
11060 ins_pipe(ialu_reg_reg_shift);
11061 %}
11062
11063 // This pattern is automatically generated from aarch64_ad.m4
11064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11065 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11066 iRegL src1, iRegL src2,
11067 immI src3) %{
11068 match(Set dst (OrL src1 (RShiftL src2 src3)));
11069
11070 ins_cost(1.9 * INSN_COST);
11071 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11072
11073 ins_encode %{
11074 __ orr(as_Register($dst$$reg),
11075 as_Register($src1$$reg),
11076 as_Register($src2$$reg),
11077 Assembler::ASR,
11078 $src3$$constant & 0x3f);
11079 %}
11080
11081 ins_pipe(ialu_reg_reg_shift);
11082 %}
11083
11084 // This pattern is automatically generated from aarch64_ad.m4
11085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11086 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11087 iRegIorL2I src1, iRegIorL2I src2,
11088 immI src3) %{
11089 match(Set dst (OrI src1 (LShiftI src2 src3)));
11090
11091 ins_cost(1.9 * INSN_COST);
11092 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11093
11094 ins_encode %{
11095 __ orrw(as_Register($dst$$reg),
11096 as_Register($src1$$reg),
11097 as_Register($src2$$reg),
11098 Assembler::LSL,
11099 $src3$$constant & 0x1f);
11100 %}
11101
11102 ins_pipe(ialu_reg_reg_shift);
11103 %}
11104
11105 // This pattern is automatically generated from aarch64_ad.m4
11106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11107 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11108 iRegL src1, iRegL src2,
11109 immI src3) %{
11110 match(Set dst (OrL src1 (LShiftL src2 src3)));
11111
11112 ins_cost(1.9 * INSN_COST);
11113 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11114
11115 ins_encode %{
11116 __ orr(as_Register($dst$$reg),
11117 as_Register($src1$$reg),
11118 as_Register($src2$$reg),
11119 Assembler::LSL,
11120 $src3$$constant & 0x3f);
11121 %}
11122
11123 ins_pipe(ialu_reg_reg_shift);
11124 %}
11125
11126 // This pattern is automatically generated from aarch64_ad.m4
11127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11128 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11129 iRegIorL2I src1, iRegIorL2I src2,
11130 immI src3) %{
11131 match(Set dst (OrI src1 (RotateRight src2 src3)));
11132
11133 ins_cost(1.9 * INSN_COST);
11134 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11135
11136 ins_encode %{
11137 __ orrw(as_Register($dst$$reg),
11138 as_Register($src1$$reg),
11139 as_Register($src2$$reg),
11140 Assembler::ROR,
11141 $src3$$constant & 0x1f);
11142 %}
11143
11144 ins_pipe(ialu_reg_reg_shift);
11145 %}
11146
11147 // This pattern is automatically generated from aarch64_ad.m4
11148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11149 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11150 iRegL src1, iRegL src2,
11151 immI src3) %{
11152 match(Set dst (OrL src1 (RotateRight src2 src3)));
11153
11154 ins_cost(1.9 * INSN_COST);
11155 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11156
11157 ins_encode %{
11158 __ orr(as_Register($dst$$reg),
11159 as_Register($src1$$reg),
11160 as_Register($src2$$reg),
11161 Assembler::ROR,
11162 $src3$$constant & 0x3f);
11163 %}
11164
11165 ins_pipe(ialu_reg_reg_shift);
11166 %}
11167
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11171 iRegIorL2I src1, iRegIorL2I src2,
11172 immI src3) %{
11173 match(Set dst (AddI src1 (URShiftI src2 src3)));
11174
11175 ins_cost(1.9 * INSN_COST);
11176 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11177
11178 ins_encode %{
11179 __ addw(as_Register($dst$$reg),
11180 as_Register($src1$$reg),
11181 as_Register($src2$$reg),
11182 Assembler::LSR,
11183 $src3$$constant & 0x1f);
11184 %}
11185
11186 ins_pipe(ialu_reg_reg_shift);
11187 %}
11188
11189 // This pattern is automatically generated from aarch64_ad.m4
11190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11191 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11192 iRegL src1, iRegL src2,
11193 immI src3) %{
11194 match(Set dst (AddL src1 (URShiftL src2 src3)));
11195
11196 ins_cost(1.9 * INSN_COST);
11197 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11198
11199 ins_encode %{
11200 __ add(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 as_Register($src2$$reg),
11203 Assembler::LSR,
11204 $src3$$constant & 0x3f);
11205 %}
11206
11207 ins_pipe(ialu_reg_reg_shift);
11208 %}
11209
11210 // This pattern is automatically generated from aarch64_ad.m4
11211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11212 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11213 iRegIorL2I src1, iRegIorL2I src2,
11214 immI src3) %{
11215 match(Set dst (AddI src1 (RShiftI src2 src3)));
11216
11217 ins_cost(1.9 * INSN_COST);
11218 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11219
11220 ins_encode %{
11221 __ addw(as_Register($dst$$reg),
11222 as_Register($src1$$reg),
11223 as_Register($src2$$reg),
11224 Assembler::ASR,
11225 $src3$$constant & 0x1f);
11226 %}
11227
11228 ins_pipe(ialu_reg_reg_shift);
11229 %}
11230
11231 // This pattern is automatically generated from aarch64_ad.m4
11232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11233 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11234 iRegL src1, iRegL src2,
11235 immI src3) %{
11236 match(Set dst (AddL src1 (RShiftL src2 src3)));
11237
11238 ins_cost(1.9 * INSN_COST);
11239 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11240
11241 ins_encode %{
11242 __ add(as_Register($dst$$reg),
11243 as_Register($src1$$reg),
11244 as_Register($src2$$reg),
11245 Assembler::ASR,
11246 $src3$$constant & 0x3f);
11247 %}
11248
11249 ins_pipe(ialu_reg_reg_shift);
11250 %}
11251
11252 // This pattern is automatically generated from aarch64_ad.m4
11253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11254 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11255 iRegIorL2I src1, iRegIorL2I src2,
11256 immI src3) %{
11257 match(Set dst (AddI src1 (LShiftI src2 src3)));
11258
11259 ins_cost(1.9 * INSN_COST);
11260 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11261
11262 ins_encode %{
11263 __ addw(as_Register($dst$$reg),
11264 as_Register($src1$$reg),
11265 as_Register($src2$$reg),
11266 Assembler::LSL,
11267 $src3$$constant & 0x1f);
11268 %}
11269
11270 ins_pipe(ialu_reg_reg_shift);
11271 %}
11272
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11276 iRegL src1, iRegL src2,
11277 immI src3) %{
11278 match(Set dst (AddL src1 (LShiftL src2 src3)));
11279
11280 ins_cost(1.9 * INSN_COST);
11281 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11282
11283 ins_encode %{
11284 __ add(as_Register($dst$$reg),
11285 as_Register($src1$$reg),
11286 as_Register($src2$$reg),
11287 Assembler::LSL,
11288 $src3$$constant & 0x3f);
11289 %}
11290
11291 ins_pipe(ialu_reg_reg_shift);
11292 %}
11293
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11297 iRegIorL2I src1, iRegIorL2I src2,
11298 immI src3) %{
11299 match(Set dst (SubI src1 (URShiftI src2 src3)));
11300
11301 ins_cost(1.9 * INSN_COST);
11302 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11303
11304 ins_encode %{
11305 __ subw(as_Register($dst$$reg),
11306 as_Register($src1$$reg),
11307 as_Register($src2$$reg),
11308 Assembler::LSR,
11309 $src3$$constant & 0x1f);
11310 %}
11311
11312 ins_pipe(ialu_reg_reg_shift);
11313 %}
11314
11315 // This pattern is automatically generated from aarch64_ad.m4
11316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11317 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11318 iRegL src1, iRegL src2,
11319 immI src3) %{
11320 match(Set dst (SubL src1 (URShiftL src2 src3)));
11321
11322 ins_cost(1.9 * INSN_COST);
11323 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11324
11325 ins_encode %{
11326 __ sub(as_Register($dst$$reg),
11327 as_Register($src1$$reg),
11328 as_Register($src2$$reg),
11329 Assembler::LSR,
11330 $src3$$constant & 0x3f);
11331 %}
11332
11333 ins_pipe(ialu_reg_reg_shift);
11334 %}
11335
11336 // This pattern is automatically generated from aarch64_ad.m4
11337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11338 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11339 iRegIorL2I src1, iRegIorL2I src2,
11340 immI src3) %{
11341 match(Set dst (SubI src1 (RShiftI src2 src3)));
11342
11343 ins_cost(1.9 * INSN_COST);
11344 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11345
11346 ins_encode %{
11347 __ subw(as_Register($dst$$reg),
11348 as_Register($src1$$reg),
11349 as_Register($src2$$reg),
11350 Assembler::ASR,
11351 $src3$$constant & 0x1f);
11352 %}
11353
11354 ins_pipe(ialu_reg_reg_shift);
11355 %}
11356
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11360 iRegL src1, iRegL src2,
11361 immI src3) %{
11362 match(Set dst (SubL src1 (RShiftL src2 src3)));
11363
11364 ins_cost(1.9 * INSN_COST);
11365 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11366
11367 ins_encode %{
11368 __ sub(as_Register($dst$$reg),
11369 as_Register($src1$$reg),
11370 as_Register($src2$$reg),
11371 Assembler::ASR,
11372 $src3$$constant & 0x3f);
11373 %}
11374
11375 ins_pipe(ialu_reg_reg_shift);
11376 %}
11377
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11381 iRegIorL2I src1, iRegIorL2I src2,
11382 immI src3) %{
11383 match(Set dst (SubI src1 (LShiftI src2 src3)));
11384
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11387
11388 ins_encode %{
11389 __ subw(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::LSL,
11393 $src3$$constant & 0x1f);
11394 %}
11395
11396 ins_pipe(ialu_reg_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11402 iRegL src1, iRegL src2,
11403 immI src3) %{
11404 match(Set dst (SubL src1 (LShiftL src2 src3)));
11405
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11408
11409 ins_encode %{
11410 __ sub(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 as_Register($src2$$reg),
11413 Assembler::LSL,
11414 $src3$$constant & 0x3f);
11415 %}
11416
11417 ins_pipe(ialu_reg_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422
11423 // Shift Left followed by Shift Right.
11424 // This idiom is used by the compiler for the i2b bytecode etc.
11425 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11426 %{
11427 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11428 ins_cost(INSN_COST * 2);
11429 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11430 ins_encode %{
11431 int lshift = $lshift_count$$constant & 63;
11432 int rshift = $rshift_count$$constant & 63;
11433 int s = 63 - lshift;
11434 int r = (rshift - lshift) & 63;
11435 __ sbfm(as_Register($dst$$reg),
11436 as_Register($src$$reg),
11437 r, s);
11438 %}
11439
11440 ins_pipe(ialu_reg_shift);
11441 %}
11442
11443 // This pattern is automatically generated from aarch64_ad.m4
11444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11445
11446 // Shift Left followed by Shift Right.
11447 // This idiom is used by the compiler for the i2b bytecode etc.
11448 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11449 %{
11450 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11451 ins_cost(INSN_COST * 2);
11452 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11453 ins_encode %{
11454 int lshift = $lshift_count$$constant & 31;
11455 int rshift = $rshift_count$$constant & 31;
11456 int s = 31 - lshift;
11457 int r = (rshift - lshift) & 31;
11458 __ sbfmw(as_Register($dst$$reg),
11459 as_Register($src$$reg),
11460 r, s);
11461 %}
11462
11463 ins_pipe(ialu_reg_shift);
11464 %}
11465
11466 // This pattern is automatically generated from aarch64_ad.m4
11467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11468
11469 // Shift Left followed by Shift Right.
11470 // This idiom is used by the compiler for the i2b bytecode etc.
11471 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11472 %{
11473 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11474 ins_cost(INSN_COST * 2);
11475 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11476 ins_encode %{
11477 int lshift = $lshift_count$$constant & 63;
11478 int rshift = $rshift_count$$constant & 63;
11479 int s = 63 - lshift;
11480 int r = (rshift - lshift) & 63;
11481 __ ubfm(as_Register($dst$$reg),
11482 as_Register($src$$reg),
11483 r, s);
11484 %}
11485
11486 ins_pipe(ialu_reg_shift);
11487 %}
11488
11489 // This pattern is automatically generated from aarch64_ad.m4
11490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11491
11492 // Shift Left followed by Shift Right.
11493 // This idiom is used by the compiler for the i2b bytecode etc.
11494 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11495 %{
11496 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11497 ins_cost(INSN_COST * 2);
11498 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11499 ins_encode %{
11500 int lshift = $lshift_count$$constant & 31;
11501 int rshift = $rshift_count$$constant & 31;
11502 int s = 31 - lshift;
11503 int r = (rshift - lshift) & 31;
11504 __ ubfmw(as_Register($dst$$reg),
11505 as_Register($src$$reg),
11506 r, s);
11507 %}
11508
11509 ins_pipe(ialu_reg_shift);
11510 %}
11511
11512 // Bitfield extract with shift & mask
11513
11514 // This pattern is automatically generated from aarch64_ad.m4
11515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11516 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11517 %{
11518 match(Set dst (AndI (URShiftI src rshift) mask));
11519 // Make sure we are not going to exceed what ubfxw can do.
11520 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11521
11522 ins_cost(INSN_COST);
11523 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11524 ins_encode %{
11525 int rshift = $rshift$$constant & 31;
11526 intptr_t mask = $mask$$constant;
11527 int width = exact_log2(mask+1);
11528 __ ubfxw(as_Register($dst$$reg),
11529 as_Register($src$$reg), rshift, width);
11530 %}
11531 ins_pipe(ialu_reg_shift);
11532 %}
11533
11534 // This pattern is automatically generated from aarch64_ad.m4
11535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11536 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11537 %{
11538 match(Set dst (AndL (URShiftL src rshift) mask));
11539 // Make sure we are not going to exceed what ubfx can do.
11540 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11541
11542 ins_cost(INSN_COST);
11543 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11544 ins_encode %{
11545 int rshift = $rshift$$constant & 63;
11546 intptr_t mask = $mask$$constant;
11547 int width = exact_log2_long(mask+1);
11548 __ ubfx(as_Register($dst$$reg),
11549 as_Register($src$$reg), rshift, width);
11550 %}
11551 ins_pipe(ialu_reg_shift);
11552 %}
11553
11554
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557
11558 // We can use ubfx when extending an And with a mask when we know mask
11559 // is positive. We know that because immI_bitmask guarantees it.
11560 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11561 %{
11562 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11563 // Make sure we are not going to exceed what ubfxw can do.
11564 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11565
11566 ins_cost(INSN_COST * 2);
11567 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11568 ins_encode %{
11569 int rshift = $rshift$$constant & 31;
11570 intptr_t mask = $mask$$constant;
11571 int width = exact_log2(mask+1);
11572 __ ubfx(as_Register($dst$$reg),
11573 as_Register($src$$reg), rshift, width);
11574 %}
11575 ins_pipe(ialu_reg_shift);
11576 %}
11577
11578
11579 // This pattern is automatically generated from aarch64_ad.m4
11580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11581
11582 // We can use ubfiz when masking by a positive number and then left shifting the result.
11583 // We know that the mask is positive because immI_bitmask guarantees it.
11584 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11585 %{
11586 match(Set dst (LShiftI (AndI src mask) lshift));
11587 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11588
11589 ins_cost(INSN_COST);
11590 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11591 ins_encode %{
11592 int lshift = $lshift$$constant & 31;
11593 intptr_t mask = $mask$$constant;
11594 int width = exact_log2(mask+1);
11595 __ ubfizw(as_Register($dst$$reg),
11596 as_Register($src$$reg), lshift, width);
11597 %}
11598 ins_pipe(ialu_reg_shift);
11599 %}
11600
11601 // This pattern is automatically generated from aarch64_ad.m4
11602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11603
11604 // We can use ubfiz when masking by a positive number and then left shifting the result.
11605 // We know that the mask is positive because immL_bitmask guarantees it.
11606 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11607 %{
11608 match(Set dst (LShiftL (AndL src mask) lshift));
11609 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11610
11611 ins_cost(INSN_COST);
11612 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11613 ins_encode %{
11614 int lshift = $lshift$$constant & 63;
11615 intptr_t mask = $mask$$constant;
11616 int width = exact_log2_long(mask+1);
11617 __ ubfiz(as_Register($dst$$reg),
11618 as_Register($src$$reg), lshift, width);
11619 %}
11620 ins_pipe(ialu_reg_shift);
11621 %}
11622
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625
11626 // We can use ubfiz when masking by a positive number and then left shifting the result.
11627 // We know that the mask is positive because immI_bitmask guarantees it.
11628 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11629 %{
11630 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11631 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11632
11633 ins_cost(INSN_COST);
11634 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11635 ins_encode %{
11636 int lshift = $lshift$$constant & 31;
11637 intptr_t mask = $mask$$constant;
11638 int width = exact_log2(mask+1);
11639 __ ubfizw(as_Register($dst$$reg),
11640 as_Register($src$$reg), lshift, width);
11641 %}
11642 ins_pipe(ialu_reg_shift);
11643 %}
11644
11645 // This pattern is automatically generated from aarch64_ad.m4
11646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11647
11648 // We can use ubfiz when masking by a positive number and then left shifting the result.
11649 // We know that the mask is positive because immL_bitmask guarantees it.
11650 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11651 %{
11652 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11653 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11654
11655 ins_cost(INSN_COST);
11656 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11657 ins_encode %{
11658 int lshift = $lshift$$constant & 63;
11659 intptr_t mask = $mask$$constant;
11660 int width = exact_log2_long(mask+1);
11661 __ ubfiz(as_Register($dst$$reg),
11662 as_Register($src$$reg), lshift, width);
11663 %}
11664 ins_pipe(ialu_reg_shift);
11665 %}
11666
11667
11668 // This pattern is automatically generated from aarch64_ad.m4
11669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11670
11671 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11672 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11673 %{
11674 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11675 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11676
11677 ins_cost(INSN_COST);
11678 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11679 ins_encode %{
11680 int lshift = $lshift$$constant & 63;
11681 intptr_t mask = $mask$$constant;
11682 int width = exact_log2(mask+1);
11683 __ ubfiz(as_Register($dst$$reg),
11684 as_Register($src$$reg), lshift, width);
11685 %}
11686 ins_pipe(ialu_reg_shift);
11687 %}
11688
11689 // This pattern is automatically generated from aarch64_ad.m4
11690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11691
11692 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11693 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11694 %{
11695 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11696 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11697
11698 ins_cost(INSN_COST);
11699 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11700 ins_encode %{
11701 int lshift = $lshift$$constant & 31;
11702 intptr_t mask = $mask$$constant;
11703 int width = exact_log2(mask+1);
11704 __ ubfiz(as_Register($dst$$reg),
11705 as_Register($src$$reg), lshift, width);
11706 %}
11707 ins_pipe(ialu_reg_shift);
11708 %}
11709
11710 // This pattern is automatically generated from aarch64_ad.m4
11711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11712
11713 // Can skip int2long conversions after AND with small bitmask
11714 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11715 %{
11716 match(Set dst (ConvI2L (AndI src msk)));
11717 ins_cost(INSN_COST);
11718 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11719 ins_encode %{
11720 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11721 %}
11722 ins_pipe(ialu_reg_shift);
11723 %}
11724
11725
11726 // Rotations
11727
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11731 %{
11732 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11733 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11734
11735 ins_cost(INSN_COST);
11736 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11737
11738 ins_encode %{
11739 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11740 $rshift$$constant & 63);
11741 %}
11742 ins_pipe(ialu_reg_reg_extr);
11743 %}
11744
11745
11746 // This pattern is automatically generated from aarch64_ad.m4
11747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11748 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11749 %{
11750 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11751 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11752
11753 ins_cost(INSN_COST);
11754 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11755
11756 ins_encode %{
11757 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11758 $rshift$$constant & 31);
11759 %}
11760 ins_pipe(ialu_reg_reg_extr);
11761 %}
11762
11763
11764 // This pattern is automatically generated from aarch64_ad.m4
11765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11766 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11767 %{
11768 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11769 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11770
11771 ins_cost(INSN_COST);
11772 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11773
11774 ins_encode %{
11775 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11776 $rshift$$constant & 63);
11777 %}
11778 ins_pipe(ialu_reg_reg_extr);
11779 %}
11780
11781
11782 // This pattern is automatically generated from aarch64_ad.m4
11783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11784 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11785 %{
11786 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11787 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11788
11789 ins_cost(INSN_COST);
11790 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11791
11792 ins_encode %{
11793 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11794 $rshift$$constant & 31);
11795 %}
11796 ins_pipe(ialu_reg_reg_extr);
11797 %}
11798
11799 // This pattern is automatically generated from aarch64_ad.m4
11800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11801 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11802 %{
11803 match(Set dst (RotateRight src shift));
11804
11805 ins_cost(INSN_COST);
11806 format %{ "ror $dst, $src, $shift" %}
11807
11808 ins_encode %{
11809 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11810 $shift$$constant & 0x1f);
11811 %}
11812 ins_pipe(ialu_reg_reg_vshift);
11813 %}
11814
11815 // This pattern is automatically generated from aarch64_ad.m4
11816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11817 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11818 %{
11819 match(Set dst (RotateRight src shift));
11820
11821 ins_cost(INSN_COST);
11822 format %{ "ror $dst, $src, $shift" %}
11823
11824 ins_encode %{
11825 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11826 $shift$$constant & 0x3f);
11827 %}
11828 ins_pipe(ialu_reg_reg_vshift);
11829 %}
11830
11831 // This pattern is automatically generated from aarch64_ad.m4
11832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11833 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11834 %{
11835 match(Set dst (RotateRight src shift));
11836
11837 ins_cost(INSN_COST);
11838 format %{ "ror $dst, $src, $shift" %}
11839
11840 ins_encode %{
11841 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11842 %}
11843 ins_pipe(ialu_reg_reg_vshift);
11844 %}
11845
11846 // This pattern is automatically generated from aarch64_ad.m4
11847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11848 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11849 %{
11850 match(Set dst (RotateRight src shift));
11851
11852 ins_cost(INSN_COST);
11853 format %{ "ror $dst, $src, $shift" %}
11854
11855 ins_encode %{
11856 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11857 %}
11858 ins_pipe(ialu_reg_reg_vshift);
11859 %}
11860
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11864 %{
11865 match(Set dst (RotateLeft src shift));
11866
11867 ins_cost(INSN_COST);
11868 format %{ "rol $dst, $src, $shift" %}
11869
11870 ins_encode %{
11871 __ subw(rscratch1, zr, as_Register($shift$$reg));
11872 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11873 %}
11874 ins_pipe(ialu_reg_reg_vshift);
11875 %}
11876
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11880 %{
11881 match(Set dst (RotateLeft src shift));
11882
11883 ins_cost(INSN_COST);
11884 format %{ "rol $dst, $src, $shift" %}
11885
11886 ins_encode %{
11887 __ subw(rscratch1, zr, as_Register($shift$$reg));
11888 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11889 %}
11890 ins_pipe(ialu_reg_reg_vshift);
11891 %}
11892
11893
11894 // Add/subtract (extended)
11895
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11899 %{
11900 match(Set dst (AddL src1 (ConvI2L src2)));
11901 ins_cost(INSN_COST);
11902 format %{ "add $dst, $src1, $src2, sxtw" %}
11903
11904 ins_encode %{
11905 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11906 as_Register($src2$$reg), ext::sxtw);
11907 %}
11908 ins_pipe(ialu_reg_reg);
11909 %}
11910
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11914 %{
11915 match(Set dst (SubL src1 (ConvI2L src2)));
11916 ins_cost(INSN_COST);
11917 format %{ "sub $dst, $src1, $src2, sxtw" %}
11918
11919 ins_encode %{
11920 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11921 as_Register($src2$$reg), ext::sxtw);
11922 %}
11923 ins_pipe(ialu_reg_reg);
11924 %}
11925
11926 // This pattern is automatically generated from aarch64_ad.m4
11927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11928 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11929 %{
11930 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11931 ins_cost(INSN_COST);
11932 format %{ "add $dst, $src1, $src2, sxth" %}
11933
11934 ins_encode %{
11935 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11936 as_Register($src2$$reg), ext::sxth);
11937 %}
11938 ins_pipe(ialu_reg_reg);
11939 %}
11940
11941 // This pattern is automatically generated from aarch64_ad.m4
11942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11943 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11944 %{
11945 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11946 ins_cost(INSN_COST);
11947 format %{ "add $dst, $src1, $src2, sxtb" %}
11948
11949 ins_encode %{
11950 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11951 as_Register($src2$$reg), ext::sxtb);
11952 %}
11953 ins_pipe(ialu_reg_reg);
11954 %}
11955
11956 // This pattern is automatically generated from aarch64_ad.m4
11957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11958 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11959 %{
11960 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11961 ins_cost(INSN_COST);
11962 format %{ "add $dst, $src1, $src2, uxtb" %}
11963
11964 ins_encode %{
11965 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11966 as_Register($src2$$reg), ext::uxtb);
11967 %}
11968 ins_pipe(ialu_reg_reg);
11969 %}
11970
11971 // This pattern is automatically generated from aarch64_ad.m4
11972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11973 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11974 %{
11975 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11976 ins_cost(INSN_COST);
11977 format %{ "add $dst, $src1, $src2, sxth" %}
11978
11979 ins_encode %{
11980 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11981 as_Register($src2$$reg), ext::sxth);
11982 %}
11983 ins_pipe(ialu_reg_reg);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11989 %{
11990 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11991 ins_cost(INSN_COST);
11992 format %{ "add $dst, $src1, $src2, sxtw" %}
11993
11994 ins_encode %{
11995 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11996 as_Register($src2$$reg), ext::sxtw);
11997 %}
11998 ins_pipe(ialu_reg_reg);
11999 %}
12000
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12004 %{
12005 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12006 ins_cost(INSN_COST);
12007 format %{ "add $dst, $src1, $src2, sxtb" %}
12008
12009 ins_encode %{
12010 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12011 as_Register($src2$$reg), ext::sxtb);
12012 %}
12013 ins_pipe(ialu_reg_reg);
12014 %}
12015
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12019 %{
12020 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12021 ins_cost(INSN_COST);
12022 format %{ "add $dst, $src1, $src2, uxtb" %}
12023
12024 ins_encode %{
12025 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12026 as_Register($src2$$reg), ext::uxtb);
12027 %}
12028 ins_pipe(ialu_reg_reg);
12029 %}
12030
12031 // This pattern is automatically generated from aarch64_ad.m4
12032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12033 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12034 %{
12035 match(Set dst (AddI src1 (AndI src2 mask)));
12036 ins_cost(INSN_COST);
12037 format %{ "addw $dst, $src1, $src2, uxtb" %}
12038
12039 ins_encode %{
12040 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12041 as_Register($src2$$reg), ext::uxtb);
12042 %}
12043 ins_pipe(ialu_reg_reg);
12044 %}
12045
12046 // This pattern is automatically generated from aarch64_ad.m4
12047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12048 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12049 %{
12050 match(Set dst (AddI src1 (AndI src2 mask)));
12051 ins_cost(INSN_COST);
12052 format %{ "addw $dst, $src1, $src2, uxth" %}
12053
12054 ins_encode %{
12055 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12056 as_Register($src2$$reg), ext::uxth);
12057 %}
12058 ins_pipe(ialu_reg_reg);
12059 %}
12060
12061 // This pattern is automatically generated from aarch64_ad.m4
12062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12063 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12064 %{
12065 match(Set dst (AddL src1 (AndL src2 mask)));
12066 ins_cost(INSN_COST);
12067 format %{ "add $dst, $src1, $src2, uxtb" %}
12068
12069 ins_encode %{
12070 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12071 as_Register($src2$$reg), ext::uxtb);
12072 %}
12073 ins_pipe(ialu_reg_reg);
12074 %}
12075
12076 // This pattern is automatically generated from aarch64_ad.m4
12077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12078 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12079 %{
12080 match(Set dst (AddL src1 (AndL src2 mask)));
12081 ins_cost(INSN_COST);
12082 format %{ "add $dst, $src1, $src2, uxth" %}
12083
12084 ins_encode %{
12085 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12086 as_Register($src2$$reg), ext::uxth);
12087 %}
12088 ins_pipe(ialu_reg_reg);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12094 %{
12095 match(Set dst (AddL src1 (AndL src2 mask)));
12096 ins_cost(INSN_COST);
12097 format %{ "add $dst, $src1, $src2, uxtw" %}
12098
12099 ins_encode %{
12100 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12101 as_Register($src2$$reg), ext::uxtw);
12102 %}
12103 ins_pipe(ialu_reg_reg);
12104 %}
12105
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12109 %{
12110 match(Set dst (SubI src1 (AndI src2 mask)));
12111 ins_cost(INSN_COST);
12112 format %{ "subw $dst, $src1, $src2, uxtb" %}
12113
12114 ins_encode %{
12115 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12116 as_Register($src2$$reg), ext::uxtb);
12117 %}
12118 ins_pipe(ialu_reg_reg);
12119 %}
12120
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12124 %{
12125 match(Set dst (SubI src1 (AndI src2 mask)));
12126 ins_cost(INSN_COST);
12127 format %{ "subw $dst, $src1, $src2, uxth" %}
12128
12129 ins_encode %{
12130 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12131 as_Register($src2$$reg), ext::uxth);
12132 %}
12133 ins_pipe(ialu_reg_reg);
12134 %}
12135
12136 // This pattern is automatically generated from aarch64_ad.m4
12137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12138 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12139 %{
12140 match(Set dst (SubL src1 (AndL src2 mask)));
12141 ins_cost(INSN_COST);
12142 format %{ "sub $dst, $src1, $src2, uxtb" %}
12143
12144 ins_encode %{
12145 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12146 as_Register($src2$$reg), ext::uxtb);
12147 %}
12148 ins_pipe(ialu_reg_reg);
12149 %}
12150
12151 // This pattern is automatically generated from aarch64_ad.m4
12152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12153 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12154 %{
12155 match(Set dst (SubL src1 (AndL src2 mask)));
12156 ins_cost(INSN_COST);
12157 format %{ "sub $dst, $src1, $src2, uxth" %}
12158
12159 ins_encode %{
12160 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12161 as_Register($src2$$reg), ext::uxth);
12162 %}
12163 ins_pipe(ialu_reg_reg);
12164 %}
12165
12166 // This pattern is automatically generated from aarch64_ad.m4
12167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12168 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12169 %{
12170 match(Set dst (SubL src1 (AndL src2 mask)));
12171 ins_cost(INSN_COST);
12172 format %{ "sub $dst, $src1, $src2, uxtw" %}
12173
12174 ins_encode %{
12175 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12176 as_Register($src2$$reg), ext::uxtw);
12177 %}
12178 ins_pipe(ialu_reg_reg);
12179 %}
12180
12181
12182 // This pattern is automatically generated from aarch64_ad.m4
12183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12184 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12185 %{
12186 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12187 ins_cost(1.9 * INSN_COST);
12188 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12189
12190 ins_encode %{
12191 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12192 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12193 %}
12194 ins_pipe(ialu_reg_reg_shift);
12195 %}
12196
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12200 %{
12201 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12202 ins_cost(1.9 * INSN_COST);
12203 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12204
12205 ins_encode %{
12206 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12207 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12208 %}
12209 ins_pipe(ialu_reg_reg_shift);
12210 %}
12211
12212 // This pattern is automatically generated from aarch64_ad.m4
12213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12214 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12215 %{
12216 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12217 ins_cost(1.9 * INSN_COST);
12218 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12219
12220 ins_encode %{
12221 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12222 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12223 %}
12224 ins_pipe(ialu_reg_reg_shift);
12225 %}
12226
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12230 %{
12231 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12232 ins_cost(1.9 * INSN_COST);
12233 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12234
12235 ins_encode %{
12236 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12237 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12238 %}
12239 ins_pipe(ialu_reg_reg_shift);
12240 %}
12241
12242 // This pattern is automatically generated from aarch64_ad.m4
12243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12244 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12245 %{
12246 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12247 ins_cost(1.9 * INSN_COST);
12248 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12249
12250 ins_encode %{
12251 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12252 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12253 %}
12254 ins_pipe(ialu_reg_reg_shift);
12255 %}
12256
12257 // This pattern is automatically generated from aarch64_ad.m4
12258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12259 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12260 %{
12261 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12262 ins_cost(1.9 * INSN_COST);
12263 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12264
12265 ins_encode %{
12266 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12267 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12268 %}
12269 ins_pipe(ialu_reg_reg_shift);
12270 %}
12271
12272 // This pattern is automatically generated from aarch64_ad.m4
12273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12274 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12275 %{
12276 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12277 ins_cost(1.9 * INSN_COST);
12278 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12279
12280 ins_encode %{
12281 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12282 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12283 %}
12284 ins_pipe(ialu_reg_reg_shift);
12285 %}
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12290 %{
12291 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12292 ins_cost(1.9 * INSN_COST);
12293 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12294
12295 ins_encode %{
12296 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12297 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12298 %}
12299 ins_pipe(ialu_reg_reg_shift);
12300 %}
12301
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12305 %{
12306 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12307 ins_cost(1.9 * INSN_COST);
12308 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12309
12310 ins_encode %{
12311 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12312 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12313 %}
12314 ins_pipe(ialu_reg_reg_shift);
12315 %}
12316
12317 // This pattern is automatically generated from aarch64_ad.m4
12318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12319 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12320 %{
12321 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12322 ins_cost(1.9 * INSN_COST);
12323 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12324
12325 ins_encode %{
12326 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12327 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12328 %}
12329 ins_pipe(ialu_reg_reg_shift);
12330 %}
12331
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12335 %{
12336 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12337 ins_cost(1.9 * INSN_COST);
12338 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12339
12340 ins_encode %{
12341 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12342 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12343 %}
12344 ins_pipe(ialu_reg_reg_shift);
12345 %}
12346
12347 // This pattern is automatically generated from aarch64_ad.m4
12348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12349 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12350 %{
12351 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12352 ins_cost(1.9 * INSN_COST);
12353 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12354
12355 ins_encode %{
12356 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12357 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12358 %}
12359 ins_pipe(ialu_reg_reg_shift);
12360 %}
12361
12362 // This pattern is automatically generated from aarch64_ad.m4
12363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12364 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12365 %{
12366 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12367 ins_cost(1.9 * INSN_COST);
12368 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12369
12370 ins_encode %{
12371 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12372 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12373 %}
12374 ins_pipe(ialu_reg_reg_shift);
12375 %}
12376
12377 // This pattern is automatically generated from aarch64_ad.m4
12378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12379 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12380 %{
12381 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12382 ins_cost(1.9 * INSN_COST);
12383 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12384
12385 ins_encode %{
12386 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12387 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12388 %}
12389 ins_pipe(ialu_reg_reg_shift);
12390 %}
12391
12392 // This pattern is automatically generated from aarch64_ad.m4
12393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12394 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12395 %{
12396 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12397 ins_cost(1.9 * INSN_COST);
12398 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12399
12400 ins_encode %{
12401 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12402 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12403 %}
12404 ins_pipe(ialu_reg_reg_shift);
12405 %}
12406
12407 // This pattern is automatically generated from aarch64_ad.m4
12408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12409 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12410 %{
12411 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12412 ins_cost(1.9 * INSN_COST);
12413 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12414
12415 ins_encode %{
12416 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12417 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12418 %}
12419 ins_pipe(ialu_reg_reg_shift);
12420 %}
12421
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12425 %{
12426 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12427 ins_cost(1.9 * INSN_COST);
12428 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12429
12430 ins_encode %{
12431 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12432 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12433 %}
12434 ins_pipe(ialu_reg_reg_shift);
12435 %}
12436
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12440 %{
12441 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12442 ins_cost(1.9 * INSN_COST);
12443 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12444
12445 ins_encode %{
12446 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12447 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12448 %}
12449 ins_pipe(ialu_reg_reg_shift);
12450 %}
12451
12452 // This pattern is automatically generated from aarch64_ad.m4
12453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12454 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12455 %{
12456 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12457 ins_cost(1.9 * INSN_COST);
12458 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12459
12460 ins_encode %{
12461 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12462 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12463 %}
12464 ins_pipe(ialu_reg_reg_shift);
12465 %}
12466
12467 // This pattern is automatically generated from aarch64_ad.m4
12468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12469 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12470 %{
12471 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12472 ins_cost(1.9 * INSN_COST);
12473 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12474
12475 ins_encode %{
12476 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12477 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12478 %}
12479 ins_pipe(ialu_reg_reg_shift);
12480 %}
12481
12482 // This pattern is automatically generated from aarch64_ad.m4
12483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12484 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12485 %{
12486 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12487 ins_cost(1.9 * INSN_COST);
12488 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12489
12490 ins_encode %{
12491 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12492 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12493 %}
12494 ins_pipe(ialu_reg_reg_shift);
12495 %}
12496
12497 // This pattern is automatically generated from aarch64_ad.m4
12498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12499 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12500 %{
12501 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12502 ins_cost(1.9 * INSN_COST);
12503 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12504
12505 ins_encode %{
12506 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12507 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12508 %}
12509 ins_pipe(ialu_reg_reg_shift);
12510 %}
12511
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12515 %{
12516 effect(DEF dst, USE src1, USE src2, USE cr);
12517 ins_cost(INSN_COST * 2);
12518 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12519
12520 ins_encode %{
12521 __ cselw($dst$$Register,
12522 $src1$$Register,
12523 $src2$$Register,
12524 Assembler::LT);
12525 %}
12526 ins_pipe(icond_reg_reg);
12527 %}
12528
12529 // This pattern is automatically generated from aarch64_ad.m4
12530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12531 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12532 %{
12533 effect(DEF dst, USE src1, USE src2, USE cr);
12534 ins_cost(INSN_COST * 2);
12535 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12536
12537 ins_encode %{
12538 __ cselw($dst$$Register,
12539 $src1$$Register,
12540 $src2$$Register,
12541 Assembler::GT);
12542 %}
12543 ins_pipe(icond_reg_reg);
12544 %}
12545
12546 // This pattern is automatically generated from aarch64_ad.m4
12547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12548 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12549 %{
12550 effect(DEF dst, USE src1, USE cr);
12551 ins_cost(INSN_COST * 2);
12552 format %{ "cselw $dst, $src1, zr lt\t" %}
12553
12554 ins_encode %{
12555 __ cselw($dst$$Register,
12556 $src1$$Register,
12557 zr,
12558 Assembler::LT);
12559 %}
12560 ins_pipe(icond_reg);
12561 %}
12562
12563 // This pattern is automatically generated from aarch64_ad.m4
12564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12565 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12566 %{
12567 effect(DEF dst, USE src1, USE cr);
12568 ins_cost(INSN_COST * 2);
12569 format %{ "cselw $dst, $src1, zr gt\t" %}
12570
12571 ins_encode %{
12572 __ cselw($dst$$Register,
12573 $src1$$Register,
12574 zr,
12575 Assembler::GT);
12576 %}
12577 ins_pipe(icond_reg);
12578 %}
12579
12580 // This pattern is automatically generated from aarch64_ad.m4
12581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12582 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12583 %{
12584 effect(DEF dst, USE src1, USE cr);
12585 ins_cost(INSN_COST * 2);
12586 format %{ "csincw $dst, $src1, zr le\t" %}
12587
12588 ins_encode %{
12589 __ csincw($dst$$Register,
12590 $src1$$Register,
12591 zr,
12592 Assembler::LE);
12593 %}
12594 ins_pipe(icond_reg);
12595 %}
12596
12597 // This pattern is automatically generated from aarch64_ad.m4
12598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12599 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12600 %{
12601 effect(DEF dst, USE src1, USE cr);
12602 ins_cost(INSN_COST * 2);
12603 format %{ "csincw $dst, $src1, zr gt\t" %}
12604
12605 ins_encode %{
12606 __ csincw($dst$$Register,
12607 $src1$$Register,
12608 zr,
12609 Assembler::GT);
12610 %}
12611 ins_pipe(icond_reg);
12612 %}
12613
12614 // This pattern is automatically generated from aarch64_ad.m4
12615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12616 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12617 %{
12618 effect(DEF dst, USE src1, USE cr);
12619 ins_cost(INSN_COST * 2);
12620 format %{ "csinvw $dst, $src1, zr lt\t" %}
12621
12622 ins_encode %{
12623 __ csinvw($dst$$Register,
12624 $src1$$Register,
12625 zr,
12626 Assembler::LT);
12627 %}
12628 ins_pipe(icond_reg);
12629 %}
12630
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12634 %{
12635 effect(DEF dst, USE src1, USE cr);
12636 ins_cost(INSN_COST * 2);
12637 format %{ "csinvw $dst, $src1, zr ge\t" %}
12638
12639 ins_encode %{
12640 __ csinvw($dst$$Register,
12641 $src1$$Register,
12642 zr,
12643 Assembler::GE);
12644 %}
12645 ins_pipe(icond_reg);
12646 %}
12647
12648 // This pattern is automatically generated from aarch64_ad.m4
12649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12650 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12651 %{
12652 match(Set dst (MinI src imm));
12653 ins_cost(INSN_COST * 3);
12654 expand %{
12655 rFlagsReg cr;
12656 compI_reg_imm0(cr, src);
12657 cmovI_reg_imm0_lt(dst, src, cr);
12658 %}
12659 %}
12660
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12664 %{
12665 match(Set dst (MinI imm src));
12666 ins_cost(INSN_COST * 3);
12667 expand %{
12668 rFlagsReg cr;
12669 compI_reg_imm0(cr, src);
12670 cmovI_reg_imm0_lt(dst, src, cr);
12671 %}
12672 %}
12673
12674 // This pattern is automatically generated from aarch64_ad.m4
12675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12676 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12677 %{
12678 match(Set dst (MinI src imm));
12679 ins_cost(INSN_COST * 3);
12680 expand %{
12681 rFlagsReg cr;
12682 compI_reg_imm0(cr, src);
12683 cmovI_reg_imm1_le(dst, src, cr);
12684 %}
12685 %}
12686
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12690 %{
12691 match(Set dst (MinI imm src));
12692 ins_cost(INSN_COST * 3);
12693 expand %{
12694 rFlagsReg cr;
12695 compI_reg_imm0(cr, src);
12696 cmovI_reg_imm1_le(dst, src, cr);
12697 %}
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12703 %{
12704 match(Set dst (MinI src imm));
12705 ins_cost(INSN_COST * 3);
12706 expand %{
12707 rFlagsReg cr;
12708 compI_reg_imm0(cr, src);
12709 cmovI_reg_immM1_lt(dst, src, cr);
12710 %}
12711 %}
12712
12713 // This pattern is automatically generated from aarch64_ad.m4
12714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12715 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12716 %{
12717 match(Set dst (MinI imm src));
12718 ins_cost(INSN_COST * 3);
12719 expand %{
12720 rFlagsReg cr;
12721 compI_reg_imm0(cr, src);
12722 cmovI_reg_immM1_lt(dst, src, cr);
12723 %}
12724 %}
12725
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12729 %{
12730 match(Set dst (MaxI src imm));
12731 ins_cost(INSN_COST * 3);
12732 expand %{
12733 rFlagsReg cr;
12734 compI_reg_imm0(cr, src);
12735 cmovI_reg_imm0_gt(dst, src, cr);
12736 %}
12737 %}
12738
12739 // This pattern is automatically generated from aarch64_ad.m4
12740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12741 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12742 %{
12743 match(Set dst (MaxI imm src));
12744 ins_cost(INSN_COST * 3);
12745 expand %{
12746 rFlagsReg cr;
12747 compI_reg_imm0(cr, src);
12748 cmovI_reg_imm0_gt(dst, src, cr);
12749 %}
12750 %}
12751
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12755 %{
12756 match(Set dst (MaxI src imm));
12757 ins_cost(INSN_COST * 3);
12758 expand %{
12759 rFlagsReg cr;
12760 compI_reg_imm0(cr, src);
12761 cmovI_reg_imm1_gt(dst, src, cr);
12762 %}
12763 %}
12764
12765 // This pattern is automatically generated from aarch64_ad.m4
12766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12767 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12768 %{
12769 match(Set dst (MaxI imm src));
12770 ins_cost(INSN_COST * 3);
12771 expand %{
12772 rFlagsReg cr;
12773 compI_reg_imm0(cr, src);
12774 cmovI_reg_imm1_gt(dst, src, cr);
12775 %}
12776 %}
12777
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12781 %{
12782 match(Set dst (MaxI src imm));
12783 ins_cost(INSN_COST * 3);
12784 expand %{
12785 rFlagsReg cr;
12786 compI_reg_imm0(cr, src);
12787 cmovI_reg_immM1_ge(dst, src, cr);
12788 %}
12789 %}
12790
12791 // This pattern is automatically generated from aarch64_ad.m4
12792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12793 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12794 %{
12795 match(Set dst (MaxI imm src));
12796 ins_cost(INSN_COST * 3);
12797 expand %{
12798 rFlagsReg cr;
12799 compI_reg_imm0(cr, src);
12800 cmovI_reg_immM1_ge(dst, src, cr);
12801 %}
12802 %}
12803
12804 // This pattern is automatically generated from aarch64_ad.m4
12805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12806 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12807 %{
12808 match(Set dst (ReverseI src));
12809 ins_cost(INSN_COST);
12810 format %{ "rbitw $dst, $src" %}
12811 ins_encode %{
12812 __ rbitw($dst$$Register, $src$$Register);
12813 %}
12814 ins_pipe(ialu_reg);
12815 %}
12816
12817 // This pattern is automatically generated from aarch64_ad.m4
12818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12819 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12820 %{
12821 match(Set dst (ReverseL src));
12822 ins_cost(INSN_COST);
12823 format %{ "rbit $dst, $src" %}
12824 ins_encode %{
12825 __ rbit($dst$$Register, $src$$Register);
12826 %}
12827 ins_pipe(ialu_reg);
12828 %}
12829
12830
12831 // END This section of the file is automatically generated. Do not edit --------------
12832
12833
12834 // ============================================================================
12835 // Floating Point Arithmetic Instructions
12836
12837 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12838 match(Set dst (AddHF src1 src2));
12839 format %{ "faddh $dst, $src1, $src2" %}
12840 ins_encode %{
12841 __ faddh($dst$$FloatRegister,
12842 $src1$$FloatRegister,
12843 $src2$$FloatRegister);
12844 %}
12845 ins_pipe(fp_dop_reg_reg_s);
12846 %}
12847
12848 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12849 match(Set dst (AddF src1 src2));
12850
12851 ins_cost(INSN_COST * 5);
12852 format %{ "fadds $dst, $src1, $src2" %}
12853
12854 ins_encode %{
12855 __ fadds(as_FloatRegister($dst$$reg),
12856 as_FloatRegister($src1$$reg),
12857 as_FloatRegister($src2$$reg));
12858 %}
12859
12860 ins_pipe(fp_dop_reg_reg_s);
12861 %}
12862
12863 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12864 match(Set dst (AddD src1 src2));
12865
12866 ins_cost(INSN_COST * 5);
12867 format %{ "faddd $dst, $src1, $src2" %}
12868
12869 ins_encode %{
12870 __ faddd(as_FloatRegister($dst$$reg),
12871 as_FloatRegister($src1$$reg),
12872 as_FloatRegister($src2$$reg));
12873 %}
12874
12875 ins_pipe(fp_dop_reg_reg_d);
12876 %}
12877
12878 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12879 match(Set dst (SubHF src1 src2));
12880 format %{ "fsubh $dst, $src1, $src2" %}
12881 ins_encode %{
12882 __ fsubh($dst$$FloatRegister,
12883 $src1$$FloatRegister,
12884 $src2$$FloatRegister);
12885 %}
12886 ins_pipe(fp_dop_reg_reg_s);
12887 %}
12888
12889 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12890 match(Set dst (SubF src1 src2));
12891
12892 ins_cost(INSN_COST * 5);
12893 format %{ "fsubs $dst, $src1, $src2" %}
12894
12895 ins_encode %{
12896 __ fsubs(as_FloatRegister($dst$$reg),
12897 as_FloatRegister($src1$$reg),
12898 as_FloatRegister($src2$$reg));
12899 %}
12900
12901 ins_pipe(fp_dop_reg_reg_s);
12902 %}
12903
12904 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12905 match(Set dst (SubD src1 src2));
12906
12907 ins_cost(INSN_COST * 5);
12908 format %{ "fsubd $dst, $src1, $src2" %}
12909
12910 ins_encode %{
12911 __ fsubd(as_FloatRegister($dst$$reg),
12912 as_FloatRegister($src1$$reg),
12913 as_FloatRegister($src2$$reg));
12914 %}
12915
12916 ins_pipe(fp_dop_reg_reg_d);
12917 %}
12918
12919 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12920 match(Set dst (MulHF src1 src2));
12921 format %{ "fmulh $dst, $src1, $src2" %}
12922 ins_encode %{
12923 __ fmulh($dst$$FloatRegister,
12924 $src1$$FloatRegister,
12925 $src2$$FloatRegister);
12926 %}
12927 ins_pipe(fp_dop_reg_reg_s);
12928 %}
12929
12930 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12931 match(Set dst (MulF src1 src2));
12932
12933 ins_cost(INSN_COST * 6);
12934 format %{ "fmuls $dst, $src1, $src2" %}
12935
12936 ins_encode %{
12937 __ fmuls(as_FloatRegister($dst$$reg),
12938 as_FloatRegister($src1$$reg),
12939 as_FloatRegister($src2$$reg));
12940 %}
12941
12942 ins_pipe(fp_dop_reg_reg_s);
12943 %}
12944
12945 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12946 match(Set dst (MulD src1 src2));
12947
12948 ins_cost(INSN_COST * 6);
12949 format %{ "fmuld $dst, $src1, $src2" %}
12950
12951 ins_encode %{
12952 __ fmuld(as_FloatRegister($dst$$reg),
12953 as_FloatRegister($src1$$reg),
12954 as_FloatRegister($src2$$reg));
12955 %}
12956
12957 ins_pipe(fp_dop_reg_reg_d);
12958 %}
12959
12960 // src1 * src2 + src3 (half-precision float)
12961 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12962 match(Set dst (FmaHF src3 (Binary src1 src2)));
12963 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12964 ins_encode %{
12965 assert(UseFMA, "Needs FMA instructions support.");
12966 __ fmaddh($dst$$FloatRegister,
12967 $src1$$FloatRegister,
12968 $src2$$FloatRegister,
12969 $src3$$FloatRegister);
12970 %}
12971 ins_pipe(pipe_class_default);
12972 %}
12973
12974 // src1 * src2 + src3
12975 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12976 match(Set dst (FmaF src3 (Binary src1 src2)));
12977
12978 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12979
12980 ins_encode %{
12981 assert(UseFMA, "Needs FMA instructions support.");
12982 __ fmadds(as_FloatRegister($dst$$reg),
12983 as_FloatRegister($src1$$reg),
12984 as_FloatRegister($src2$$reg),
12985 as_FloatRegister($src3$$reg));
12986 %}
12987
12988 ins_pipe(pipe_class_default);
12989 %}
12990
12991 // src1 * src2 + src3
12992 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12993 match(Set dst (FmaD src3 (Binary src1 src2)));
12994
12995 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12996
12997 ins_encode %{
12998 assert(UseFMA, "Needs FMA instructions support.");
12999 __ fmaddd(as_FloatRegister($dst$$reg),
13000 as_FloatRegister($src1$$reg),
13001 as_FloatRegister($src2$$reg),
13002 as_FloatRegister($src3$$reg));
13003 %}
13004
13005 ins_pipe(pipe_class_default);
13006 %}
13007
13008 // src1 * (-src2) + src3
13009 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13010 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13011 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13012
13013 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13014
13015 ins_encode %{
13016 assert(UseFMA, "Needs FMA instructions support.");
13017 __ fmsubs(as_FloatRegister($dst$$reg),
13018 as_FloatRegister($src1$$reg),
13019 as_FloatRegister($src2$$reg),
13020 as_FloatRegister($src3$$reg));
13021 %}
13022
13023 ins_pipe(pipe_class_default);
13024 %}
13025
13026 // src1 * (-src2) + src3
13027 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13028 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13029 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13030
13031 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13032
13033 ins_encode %{
13034 assert(UseFMA, "Needs FMA instructions support.");
13035 __ fmsubd(as_FloatRegister($dst$$reg),
13036 as_FloatRegister($src1$$reg),
13037 as_FloatRegister($src2$$reg),
13038 as_FloatRegister($src3$$reg));
13039 %}
13040
13041 ins_pipe(pipe_class_default);
13042 %}
13043
13044 // src1 * (-src2) - src3
13045 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13046 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13047 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13048
13049 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13050
13051 ins_encode %{
13052 assert(UseFMA, "Needs FMA instructions support.");
13053 __ fnmadds(as_FloatRegister($dst$$reg),
13054 as_FloatRegister($src1$$reg),
13055 as_FloatRegister($src2$$reg),
13056 as_FloatRegister($src3$$reg));
13057 %}
13058
13059 ins_pipe(pipe_class_default);
13060 %}
13061
13062 // src1 * (-src2) - src3
13063 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13064 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13065 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13066
13067 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13068
13069 ins_encode %{
13070 assert(UseFMA, "Needs FMA instructions support.");
13071 __ fnmaddd(as_FloatRegister($dst$$reg),
13072 as_FloatRegister($src1$$reg),
13073 as_FloatRegister($src2$$reg),
13074 as_FloatRegister($src3$$reg));
13075 %}
13076
13077 ins_pipe(pipe_class_default);
13078 %}
13079
13080 // src1 * src2 - src3
13081 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13082 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13083
13084 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13085
13086 ins_encode %{
13087 assert(UseFMA, "Needs FMA instructions support.");
13088 __ fnmsubs(as_FloatRegister($dst$$reg),
13089 as_FloatRegister($src1$$reg),
13090 as_FloatRegister($src2$$reg),
13091 as_FloatRegister($src3$$reg));
13092 %}
13093
13094 ins_pipe(pipe_class_default);
13095 %}
13096
13097 // src1 * src2 - src3
13098 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13099 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13100
13101 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13102
13103 ins_encode %{
13104 assert(UseFMA, "Needs FMA instructions support.");
13105 // n.b. insn name should be fnmsubd
13106 __ fnmsub(as_FloatRegister($dst$$reg),
13107 as_FloatRegister($src1$$reg),
13108 as_FloatRegister($src2$$reg),
13109 as_FloatRegister($src3$$reg));
13110 %}
13111
13112 ins_pipe(pipe_class_default);
13113 %}
13114
13115 // Math.max(HH)H (half-precision float)
13116 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13117 match(Set dst (MaxHF src1 src2));
13118 format %{ "fmaxh $dst, $src1, $src2" %}
13119 ins_encode %{
13120 __ fmaxh($dst$$FloatRegister,
13121 $src1$$FloatRegister,
13122 $src2$$FloatRegister);
13123 %}
13124 ins_pipe(fp_dop_reg_reg_s);
13125 %}
13126
13127 // Math.min(HH)H (half-precision float)
13128 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13129 match(Set dst (MinHF src1 src2));
13130 format %{ "fminh $dst, $src1, $src2" %}
13131 ins_encode %{
13132 __ fminh($dst$$FloatRegister,
13133 $src1$$FloatRegister,
13134 $src2$$FloatRegister);
13135 %}
13136 ins_pipe(fp_dop_reg_reg_s);
13137 %}
13138
13139 // Math.max(FF)F
13140 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13141 match(Set dst (MaxF src1 src2));
13142
13143 format %{ "fmaxs $dst, $src1, $src2" %}
13144 ins_encode %{
13145 __ fmaxs(as_FloatRegister($dst$$reg),
13146 as_FloatRegister($src1$$reg),
13147 as_FloatRegister($src2$$reg));
13148 %}
13149
13150 ins_pipe(fp_dop_reg_reg_s);
13151 %}
13152
13153 // Math.min(FF)F
13154 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13155 match(Set dst (MinF src1 src2));
13156
13157 format %{ "fmins $dst, $src1, $src2" %}
13158 ins_encode %{
13159 __ fmins(as_FloatRegister($dst$$reg),
13160 as_FloatRegister($src1$$reg),
13161 as_FloatRegister($src2$$reg));
13162 %}
13163
13164 ins_pipe(fp_dop_reg_reg_s);
13165 %}
13166
13167 // Math.max(DD)D
13168 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13169 match(Set dst (MaxD src1 src2));
13170
13171 format %{ "fmaxd $dst, $src1, $src2" %}
13172 ins_encode %{
13173 __ fmaxd(as_FloatRegister($dst$$reg),
13174 as_FloatRegister($src1$$reg),
13175 as_FloatRegister($src2$$reg));
13176 %}
13177
13178 ins_pipe(fp_dop_reg_reg_d);
13179 %}
13180
13181 // Math.min(DD)D
13182 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13183 match(Set dst (MinD src1 src2));
13184
13185 format %{ "fmind $dst, $src1, $src2" %}
13186 ins_encode %{
13187 __ fmind(as_FloatRegister($dst$$reg),
13188 as_FloatRegister($src1$$reg),
13189 as_FloatRegister($src2$$reg));
13190 %}
13191
13192 ins_pipe(fp_dop_reg_reg_d);
13193 %}
13194
13195 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13196 match(Set dst (DivHF src1 src2));
13197 format %{ "fdivh $dst, $src1, $src2" %}
13198 ins_encode %{
13199 __ fdivh($dst$$FloatRegister,
13200 $src1$$FloatRegister,
13201 $src2$$FloatRegister);
13202 %}
13203 ins_pipe(fp_div_s);
13204 %}
13205
13206 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13207 match(Set dst (DivF src1 src2));
13208
13209 ins_cost(INSN_COST * 18);
13210 format %{ "fdivs $dst, $src1, $src2" %}
13211
13212 ins_encode %{
13213 __ fdivs(as_FloatRegister($dst$$reg),
13214 as_FloatRegister($src1$$reg),
13215 as_FloatRegister($src2$$reg));
13216 %}
13217
13218 ins_pipe(fp_div_s);
13219 %}
13220
13221 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13222 match(Set dst (DivD src1 src2));
13223
13224 ins_cost(INSN_COST * 32);
13225 format %{ "fdivd $dst, $src1, $src2" %}
13226
13227 ins_encode %{
13228 __ fdivd(as_FloatRegister($dst$$reg),
13229 as_FloatRegister($src1$$reg),
13230 as_FloatRegister($src2$$reg));
13231 %}
13232
13233 ins_pipe(fp_div_d);
13234 %}
13235
13236 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13237 match(Set dst (NegF src));
13238
13239 ins_cost(INSN_COST * 3);
13240 format %{ "fneg $dst, $src" %}
13241
13242 ins_encode %{
13243 __ fnegs(as_FloatRegister($dst$$reg),
13244 as_FloatRegister($src$$reg));
13245 %}
13246
13247 ins_pipe(fp_uop_s);
13248 %}
13249
13250 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13251 match(Set dst (NegD src));
13252
13253 ins_cost(INSN_COST * 3);
13254 format %{ "fnegd $dst, $src" %}
13255
13256 ins_encode %{
13257 __ fnegd(as_FloatRegister($dst$$reg),
13258 as_FloatRegister($src$$reg));
13259 %}
13260
13261 ins_pipe(fp_uop_d);
13262 %}
13263
13264 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13265 %{
13266 match(Set dst (AbsI src));
13267
13268 effect(KILL cr);
13269 ins_cost(INSN_COST * 2);
13270 format %{ "cmpw $src, zr\n\t"
13271 "cnegw $dst, $src, Assembler::LT\t# int abs"
13272 %}
13273
13274 ins_encode %{
13275 __ cmpw(as_Register($src$$reg), zr);
13276 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13277 %}
13278 ins_pipe(pipe_class_default);
13279 %}
13280
13281 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13282 %{
13283 match(Set dst (AbsL src));
13284
13285 effect(KILL cr);
13286 ins_cost(INSN_COST * 2);
13287 format %{ "cmp $src, zr\n\t"
13288 "cneg $dst, $src, Assembler::LT\t# long abs"
13289 %}
13290
13291 ins_encode %{
13292 __ cmp(as_Register($src$$reg), zr);
13293 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13294 %}
13295 ins_pipe(pipe_class_default);
13296 %}
13297
13298 instruct absF_reg(vRegF dst, vRegF src) %{
13299 match(Set dst (AbsF src));
13300
13301 ins_cost(INSN_COST * 3);
13302 format %{ "fabss $dst, $src" %}
13303 ins_encode %{
13304 __ fabss(as_FloatRegister($dst$$reg),
13305 as_FloatRegister($src$$reg));
13306 %}
13307
13308 ins_pipe(fp_uop_s);
13309 %}
13310
13311 instruct absD_reg(vRegD dst, vRegD src) %{
13312 match(Set dst (AbsD src));
13313
13314 ins_cost(INSN_COST * 3);
13315 format %{ "fabsd $dst, $src" %}
13316 ins_encode %{
13317 __ fabsd(as_FloatRegister($dst$$reg),
13318 as_FloatRegister($src$$reg));
13319 %}
13320
13321 ins_pipe(fp_uop_d);
13322 %}
13323
13324 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13325 match(Set dst (AbsF (SubF src1 src2)));
13326
13327 ins_cost(INSN_COST * 3);
13328 format %{ "fabds $dst, $src1, $src2" %}
13329 ins_encode %{
13330 __ fabds(as_FloatRegister($dst$$reg),
13331 as_FloatRegister($src1$$reg),
13332 as_FloatRegister($src2$$reg));
13333 %}
13334
13335 ins_pipe(fp_uop_s);
13336 %}
13337
13338 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13339 match(Set dst (AbsD (SubD src1 src2)));
13340
13341 ins_cost(INSN_COST * 3);
13342 format %{ "fabdd $dst, $src1, $src2" %}
13343 ins_encode %{
13344 __ fabdd(as_FloatRegister($dst$$reg),
13345 as_FloatRegister($src1$$reg),
13346 as_FloatRegister($src2$$reg));
13347 %}
13348
13349 ins_pipe(fp_uop_d);
13350 %}
13351
13352 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13353 match(Set dst (SqrtD src));
13354
13355 ins_cost(INSN_COST * 50);
13356 format %{ "fsqrtd $dst, $src" %}
13357 ins_encode %{
13358 __ fsqrtd(as_FloatRegister($dst$$reg),
13359 as_FloatRegister($src$$reg));
13360 %}
13361
13362 ins_pipe(fp_div_s);
13363 %}
13364
13365 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13366 match(Set dst (SqrtF src));
13367
13368 ins_cost(INSN_COST * 50);
13369 format %{ "fsqrts $dst, $src" %}
13370 ins_encode %{
13371 __ fsqrts(as_FloatRegister($dst$$reg),
13372 as_FloatRegister($src$$reg));
13373 %}
13374
13375 ins_pipe(fp_div_d);
13376 %}
13377
13378 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13379 match(Set dst (SqrtHF src));
13380 format %{ "fsqrth $dst, $src" %}
13381 ins_encode %{
13382 __ fsqrth($dst$$FloatRegister,
13383 $src$$FloatRegister);
13384 %}
13385 ins_pipe(fp_div_s);
13386 %}
13387
13388 // Math.rint, floor, ceil
13389 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13390 match(Set dst (RoundDoubleMode src rmode));
13391 format %{ "frint $dst, $src, $rmode" %}
13392 ins_encode %{
13393 switch ($rmode$$constant) {
13394 case RoundDoubleModeNode::rmode_rint:
13395 __ frintnd(as_FloatRegister($dst$$reg),
13396 as_FloatRegister($src$$reg));
13397 break;
13398 case RoundDoubleModeNode::rmode_floor:
13399 __ frintmd(as_FloatRegister($dst$$reg),
13400 as_FloatRegister($src$$reg));
13401 break;
13402 case RoundDoubleModeNode::rmode_ceil:
13403 __ frintpd(as_FloatRegister($dst$$reg),
13404 as_FloatRegister($src$$reg));
13405 break;
13406 }
13407 %}
13408 ins_pipe(fp_uop_d);
13409 %}
13410
13411 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13412 match(Set dst (CopySignD src1 (Binary src2 zero)));
13413 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13414 format %{ "CopySignD $dst $src1 $src2" %}
13415 ins_encode %{
13416 FloatRegister dst = as_FloatRegister($dst$$reg),
13417 src1 = as_FloatRegister($src1$$reg),
13418 src2 = as_FloatRegister($src2$$reg),
13419 zero = as_FloatRegister($zero$$reg);
13420 __ fnegd(dst, zero);
13421 __ bsl(dst, __ T8B, src2, src1);
13422 %}
13423 ins_pipe(fp_uop_d);
13424 %}
13425
13426 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13427 match(Set dst (CopySignF src1 src2));
13428 effect(TEMP_DEF dst, USE src1, USE src2);
13429 format %{ "CopySignF $dst $src1 $src2" %}
13430 ins_encode %{
13431 FloatRegister dst = as_FloatRegister($dst$$reg),
13432 src1 = as_FloatRegister($src1$$reg),
13433 src2 = as_FloatRegister($src2$$reg);
13434 __ movi(dst, __ T2S, 0x80, 24);
13435 __ bsl(dst, __ T8B, src2, src1);
13436 %}
13437 ins_pipe(fp_uop_d);
13438 %}
13439
13440 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13441 match(Set dst (SignumD src (Binary zero one)));
13442 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13443 format %{ "signumD $dst, $src" %}
13444 ins_encode %{
13445 FloatRegister src = as_FloatRegister($src$$reg),
13446 dst = as_FloatRegister($dst$$reg),
13447 zero = as_FloatRegister($zero$$reg),
13448 one = as_FloatRegister($one$$reg);
13449 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13450 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13451 // Bit selection instruction gets bit from "one" for each enabled bit in
13452 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13453 // NaN the whole "src" will be copied because "dst" is zero. For all other
13454 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13455 // from "src", and all other bits are copied from 1.0.
13456 __ bsl(dst, __ T8B, one, src);
13457 %}
13458 ins_pipe(fp_uop_d);
13459 %}
13460
13461 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13462 match(Set dst (SignumF src (Binary zero one)));
13463 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13464 format %{ "signumF $dst, $src" %}
13465 ins_encode %{
13466 FloatRegister src = as_FloatRegister($src$$reg),
13467 dst = as_FloatRegister($dst$$reg),
13468 zero = as_FloatRegister($zero$$reg),
13469 one = as_FloatRegister($one$$reg);
13470 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13471 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13472 // Bit selection instruction gets bit from "one" for each enabled bit in
13473 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13474 // NaN the whole "src" will be copied because "dst" is zero. For all other
13475 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13476 // from "src", and all other bits are copied from 1.0.
13477 __ bsl(dst, __ T8B, one, src);
13478 %}
13479 ins_pipe(fp_uop_d);
13480 %}
13481
13482 instruct onspinwait() %{
13483 match(OnSpinWait);
13484 ins_cost(INSN_COST);
13485
13486 format %{ "onspinwait" %}
13487
13488 ins_encode %{
13489 __ spin_wait();
13490 %}
13491 ins_pipe(pipe_class_empty);
13492 %}
13493
13494 // ============================================================================
13495 // Logical Instructions
13496
13497 // Integer Logical Instructions
13498
13499 // And Instructions
13500
13501
13502 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13503 match(Set dst (AndI src1 src2));
13504
13505 format %{ "andw $dst, $src1, $src2\t# int" %}
13506
13507 ins_cost(INSN_COST);
13508 ins_encode %{
13509 __ andw(as_Register($dst$$reg),
13510 as_Register($src1$$reg),
13511 as_Register($src2$$reg));
13512 %}
13513
13514 ins_pipe(ialu_reg_reg);
13515 %}
13516
13517 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13518 match(Set dst (AndI src1 src2));
13519
13520 format %{ "andsw $dst, $src1, $src2\t# int" %}
13521
13522 ins_cost(INSN_COST);
13523 ins_encode %{
13524 __ andw(as_Register($dst$$reg),
13525 as_Register($src1$$reg),
13526 (uint64_t)($src2$$constant));
13527 %}
13528
13529 ins_pipe(ialu_reg_imm);
13530 %}
13531
13532 // Or Instructions
13533
13534 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13535 match(Set dst (OrI src1 src2));
13536
13537 format %{ "orrw $dst, $src1, $src2\t# int" %}
13538
13539 ins_cost(INSN_COST);
13540 ins_encode %{
13541 __ orrw(as_Register($dst$$reg),
13542 as_Register($src1$$reg),
13543 as_Register($src2$$reg));
13544 %}
13545
13546 ins_pipe(ialu_reg_reg);
13547 %}
13548
13549 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13550 match(Set dst (OrI src1 src2));
13551
13552 format %{ "orrw $dst, $src1, $src2\t# int" %}
13553
13554 ins_cost(INSN_COST);
13555 ins_encode %{
13556 __ orrw(as_Register($dst$$reg),
13557 as_Register($src1$$reg),
13558 (uint64_t)($src2$$constant));
13559 %}
13560
13561 ins_pipe(ialu_reg_imm);
13562 %}
13563
13564 // Xor Instructions
13565
13566 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13567 match(Set dst (XorI src1 src2));
13568
13569 format %{ "eorw $dst, $src1, $src2\t# int" %}
13570
13571 ins_cost(INSN_COST);
13572 ins_encode %{
13573 __ eorw(as_Register($dst$$reg),
13574 as_Register($src1$$reg),
13575 as_Register($src2$$reg));
13576 %}
13577
13578 ins_pipe(ialu_reg_reg);
13579 %}
13580
13581 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13582 match(Set dst (XorI src1 src2));
13583
13584 format %{ "eorw $dst, $src1, $src2\t# int" %}
13585
13586 ins_cost(INSN_COST);
13587 ins_encode %{
13588 __ eorw(as_Register($dst$$reg),
13589 as_Register($src1$$reg),
13590 (uint64_t)($src2$$constant));
13591 %}
13592
13593 ins_pipe(ialu_reg_imm);
13594 %}
13595
13596 // Long Logical Instructions
13597 // TODO
13598
13599 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13600 match(Set dst (AndL src1 src2));
13601
13602 format %{ "and $dst, $src1, $src2\t# int" %}
13603
13604 ins_cost(INSN_COST);
13605 ins_encode %{
13606 __ andr(as_Register($dst$$reg),
13607 as_Register($src1$$reg),
13608 as_Register($src2$$reg));
13609 %}
13610
13611 ins_pipe(ialu_reg_reg);
13612 %}
13613
13614 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13615 match(Set dst (AndL src1 src2));
13616
13617 format %{ "and $dst, $src1, $src2\t# int" %}
13618
13619 ins_cost(INSN_COST);
13620 ins_encode %{
13621 __ andr(as_Register($dst$$reg),
13622 as_Register($src1$$reg),
13623 (uint64_t)($src2$$constant));
13624 %}
13625
13626 ins_pipe(ialu_reg_imm);
13627 %}
13628
13629 // Or Instructions
13630
13631 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13632 match(Set dst (OrL src1 src2));
13633
13634 format %{ "orr $dst, $src1, $src2\t# int" %}
13635
13636 ins_cost(INSN_COST);
13637 ins_encode %{
13638 __ orr(as_Register($dst$$reg),
13639 as_Register($src1$$reg),
13640 as_Register($src2$$reg));
13641 %}
13642
13643 ins_pipe(ialu_reg_reg);
13644 %}
13645
13646 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13647 match(Set dst (OrL src1 src2));
13648
13649 format %{ "orr $dst, $src1, $src2\t# int" %}
13650
13651 ins_cost(INSN_COST);
13652 ins_encode %{
13653 __ orr(as_Register($dst$$reg),
13654 as_Register($src1$$reg),
13655 (uint64_t)($src2$$constant));
13656 %}
13657
13658 ins_pipe(ialu_reg_imm);
13659 %}
13660
13661 // Xor Instructions
13662
13663 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13664 match(Set dst (XorL src1 src2));
13665
13666 format %{ "eor $dst, $src1, $src2\t# int" %}
13667
13668 ins_cost(INSN_COST);
13669 ins_encode %{
13670 __ eor(as_Register($dst$$reg),
13671 as_Register($src1$$reg),
13672 as_Register($src2$$reg));
13673 %}
13674
13675 ins_pipe(ialu_reg_reg);
13676 %}
13677
13678 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13679 match(Set dst (XorL src1 src2));
13680
13681 ins_cost(INSN_COST);
13682 format %{ "eor $dst, $src1, $src2\t# int" %}
13683
13684 ins_encode %{
13685 __ eor(as_Register($dst$$reg),
13686 as_Register($src1$$reg),
13687 (uint64_t)($src2$$constant));
13688 %}
13689
13690 ins_pipe(ialu_reg_imm);
13691 %}
13692
13693 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13694 %{
13695 match(Set dst (ConvI2L src));
13696
13697 ins_cost(INSN_COST);
13698 format %{ "sxtw $dst, $src\t# i2l" %}
13699 ins_encode %{
13700 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13701 %}
13702 ins_pipe(ialu_reg_shift);
13703 %}
13704
13705 // this pattern occurs in bigmath arithmetic
13706 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13707 %{
13708 match(Set dst (AndL (ConvI2L src) mask));
13709
13710 ins_cost(INSN_COST);
13711 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13712 ins_encode %{
13713 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13714 %}
13715
13716 ins_pipe(ialu_reg_shift);
13717 %}
13718
13719 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13720 match(Set dst (ConvL2I src));
13721
13722 ins_cost(INSN_COST);
13723 format %{ "movw $dst, $src \t// l2i" %}
13724
13725 ins_encode %{
13726 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13727 %}
13728
13729 ins_pipe(ialu_reg);
13730 %}
13731
13732 instruct convD2F_reg(vRegF dst, vRegD src) %{
13733 match(Set dst (ConvD2F src));
13734
13735 ins_cost(INSN_COST * 5);
13736 format %{ "fcvtd $dst, $src \t// d2f" %}
13737
13738 ins_encode %{
13739 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13740 %}
13741
13742 ins_pipe(fp_d2f);
13743 %}
13744
13745 instruct convF2D_reg(vRegD dst, vRegF src) %{
13746 match(Set dst (ConvF2D src));
13747
13748 ins_cost(INSN_COST * 5);
13749 format %{ "fcvts $dst, $src \t// f2d" %}
13750
13751 ins_encode %{
13752 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13753 %}
13754
13755 ins_pipe(fp_f2d);
13756 %}
13757
13758 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13759 match(Set dst (ConvF2I src));
13760
13761 ins_cost(INSN_COST * 5);
13762 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13763
13764 ins_encode %{
13765 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13766 %}
13767
13768 ins_pipe(fp_f2i);
13769 %}
13770
13771 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13772 match(Set dst (ConvF2L src));
13773
13774 ins_cost(INSN_COST * 5);
13775 format %{ "fcvtzs $dst, $src \t// f2l" %}
13776
13777 ins_encode %{
13778 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13779 %}
13780
13781 ins_pipe(fp_f2l);
13782 %}
13783
13784 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13785 match(Set dst (ConvF2HF src));
13786 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13787 "smov $dst, $tmp\t# move result from $tmp to $dst"
13788 %}
13789 effect(TEMP tmp);
13790 ins_encode %{
13791 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13792 %}
13793 ins_pipe(pipe_slow);
13794 %}
13795
13796 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13797 match(Set dst (ConvHF2F src));
13798 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13799 "fcvt $dst, $tmp\t# convert half to single precision"
13800 %}
13801 effect(TEMP tmp);
13802 ins_encode %{
13803 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13804 %}
13805 ins_pipe(pipe_slow);
13806 %}
13807
13808 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13809 match(Set dst (ConvI2F src));
13810
13811 ins_cost(INSN_COST * 5);
13812 format %{ "scvtfws $dst, $src \t// i2f" %}
13813
13814 ins_encode %{
13815 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13816 %}
13817
13818 ins_pipe(fp_i2f);
13819 %}
13820
13821 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13822 match(Set dst (ConvL2F src));
13823
13824 ins_cost(INSN_COST * 5);
13825 format %{ "scvtfs $dst, $src \t// l2f" %}
13826
13827 ins_encode %{
13828 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13829 %}
13830
13831 ins_pipe(fp_l2f);
13832 %}
13833
13834 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13835 match(Set dst (ConvD2I src));
13836
13837 ins_cost(INSN_COST * 5);
13838 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13839
13840 ins_encode %{
13841 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13842 %}
13843
13844 ins_pipe(fp_d2i);
13845 %}
13846
13847 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13848 match(Set dst (ConvD2L src));
13849
13850 ins_cost(INSN_COST * 5);
13851 format %{ "fcvtzd $dst, $src \t// d2l" %}
13852
13853 ins_encode %{
13854 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13855 %}
13856
13857 ins_pipe(fp_d2l);
13858 %}
13859
13860 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13861 match(Set dst (ConvI2D src));
13862
13863 ins_cost(INSN_COST * 5);
13864 format %{ "scvtfwd $dst, $src \t// i2d" %}
13865
13866 ins_encode %{
13867 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13868 %}
13869
13870 ins_pipe(fp_i2d);
13871 %}
13872
13873 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13874 match(Set dst (ConvL2D src));
13875
13876 ins_cost(INSN_COST * 5);
13877 format %{ "scvtfd $dst, $src \t// l2d" %}
13878
13879 ins_encode %{
13880 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13881 %}
13882
13883 ins_pipe(fp_l2d);
13884 %}
13885
13886 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13887 %{
13888 match(Set dst (RoundD src));
13889 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13890 format %{ "java_round_double $dst,$src"%}
13891 ins_encode %{
13892 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13893 as_FloatRegister($ftmp$$reg));
13894 %}
13895 ins_pipe(pipe_slow);
13896 %}
13897
13898 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13899 %{
13900 match(Set dst (RoundF src));
13901 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13902 format %{ "java_round_float $dst,$src"%}
13903 ins_encode %{
13904 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13905 as_FloatRegister($ftmp$$reg));
13906 %}
13907 ins_pipe(pipe_slow);
13908 %}
13909
13910 // stack <-> reg and reg <-> reg shuffles with no conversion
13911
13912 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13913
13914 match(Set dst (MoveF2I src));
13915
13916 effect(DEF dst, USE src);
13917
13918 ins_cost(4 * INSN_COST);
13919
13920 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13921
13922 ins_encode %{
13923 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13924 %}
13925
13926 ins_pipe(iload_reg_reg);
13927
13928 %}
13929
13930 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13931
13932 match(Set dst (MoveI2F src));
13933
13934 effect(DEF dst, USE src);
13935
13936 ins_cost(4 * INSN_COST);
13937
13938 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13939
13940 ins_encode %{
13941 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13942 %}
13943
13944 ins_pipe(pipe_class_memory);
13945
13946 %}
13947
13948 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13949
13950 match(Set dst (MoveD2L src));
13951
13952 effect(DEF dst, USE src);
13953
13954 ins_cost(4 * INSN_COST);
13955
13956 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13957
13958 ins_encode %{
13959 __ ldr($dst$$Register, Address(sp, $src$$disp));
13960 %}
13961
13962 ins_pipe(iload_reg_reg);
13963
13964 %}
13965
13966 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13967
13968 match(Set dst (MoveL2D src));
13969
13970 effect(DEF dst, USE src);
13971
13972 ins_cost(4 * INSN_COST);
13973
13974 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13975
13976 ins_encode %{
13977 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13978 %}
13979
13980 ins_pipe(pipe_class_memory);
13981
13982 %}
13983
13984 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13985
13986 match(Set dst (MoveF2I src));
13987
13988 effect(DEF dst, USE src);
13989
13990 ins_cost(INSN_COST);
13991
13992 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13993
13994 ins_encode %{
13995 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13996 %}
13997
13998 ins_pipe(pipe_class_memory);
13999
14000 %}
14001
14002 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14003
14004 match(Set dst (MoveI2F src));
14005
14006 effect(DEF dst, USE src);
14007
14008 ins_cost(INSN_COST);
14009
14010 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14011
14012 ins_encode %{
14013 __ strw($src$$Register, Address(sp, $dst$$disp));
14014 %}
14015
14016 ins_pipe(istore_reg_reg);
14017
14018 %}
14019
14020 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14021
14022 match(Set dst (MoveD2L src));
14023
14024 effect(DEF dst, USE src);
14025
14026 ins_cost(INSN_COST);
14027
14028 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14029
14030 ins_encode %{
14031 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14032 %}
14033
14034 ins_pipe(pipe_class_memory);
14035
14036 %}
14037
14038 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14039
14040 match(Set dst (MoveL2D src));
14041
14042 effect(DEF dst, USE src);
14043
14044 ins_cost(INSN_COST);
14045
14046 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14047
14048 ins_encode %{
14049 __ str($src$$Register, Address(sp, $dst$$disp));
14050 %}
14051
14052 ins_pipe(istore_reg_reg);
14053
14054 %}
14055
14056 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14057
14058 match(Set dst (MoveF2I src));
14059
14060 effect(DEF dst, USE src);
14061
14062 ins_cost(INSN_COST);
14063
14064 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14065
14066 ins_encode %{
14067 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14068 %}
14069
14070 ins_pipe(fp_f2i);
14071
14072 %}
14073
14074 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14075
14076 match(Set dst (MoveI2F src));
14077
14078 effect(DEF dst, USE src);
14079
14080 ins_cost(INSN_COST);
14081
14082 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14083
14084 ins_encode %{
14085 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14086 %}
14087
14088 ins_pipe(fp_i2f);
14089
14090 %}
14091
14092 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14093
14094 match(Set dst (MoveD2L src));
14095
14096 effect(DEF dst, USE src);
14097
14098 ins_cost(INSN_COST);
14099
14100 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14101
14102 ins_encode %{
14103 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14104 %}
14105
14106 ins_pipe(fp_d2l);
14107
14108 %}
14109
14110 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14111
14112 match(Set dst (MoveL2D src));
14113
14114 effect(DEF dst, USE src);
14115
14116 ins_cost(INSN_COST);
14117
14118 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14119
14120 ins_encode %{
14121 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14122 %}
14123
14124 ins_pipe(fp_l2d);
14125
14126 %}
14127
14128 // ============================================================================
14129 // clearing of an array
14130
14131 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14132 %{
14133 match(Set dummy (ClearArray cnt base));
14134 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14135
14136 ins_cost(4 * INSN_COST);
14137 format %{ "ClearArray $cnt, $base" %}
14138
14139 ins_encode %{
14140 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14141 if (tpc == nullptr) {
14142 ciEnv::current()->record_failure("CodeCache is full");
14143 return;
14144 }
14145 %}
14146
14147 ins_pipe(pipe_class_memory);
14148 %}
14149
14150 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14151 %{
14152 predicate((uint64_t)n->in(2)->get_long()
14153 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14154 match(Set dummy (ClearArray cnt base));
14155 effect(TEMP temp, USE_KILL base, KILL cr);
14156
14157 ins_cost(4 * INSN_COST);
14158 format %{ "ClearArray $cnt, $base" %}
14159
14160 ins_encode %{
14161 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14162 if (tpc == nullptr) {
14163 ciEnv::current()->record_failure("CodeCache is full");
14164 return;
14165 }
14166 %}
14167
14168 ins_pipe(pipe_class_memory);
14169 %}
14170
14171 // ============================================================================
14172 // Overflow Math Instructions
14173
14174 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14175 %{
14176 match(Set cr (OverflowAddI op1 op2));
14177
14178 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14179 ins_cost(INSN_COST);
14180 ins_encode %{
14181 __ cmnw($op1$$Register, $op2$$Register);
14182 %}
14183
14184 ins_pipe(icmp_reg_reg);
14185 %}
14186
14187 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14188 %{
14189 match(Set cr (OverflowAddI op1 op2));
14190
14191 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14192 ins_cost(INSN_COST);
14193 ins_encode %{
14194 __ cmnw($op1$$Register, $op2$$constant);
14195 %}
14196
14197 ins_pipe(icmp_reg_imm);
14198 %}
14199
14200 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14201 %{
14202 match(Set cr (OverflowAddL op1 op2));
14203
14204 format %{ "cmn $op1, $op2\t# overflow check long" %}
14205 ins_cost(INSN_COST);
14206 ins_encode %{
14207 __ cmn($op1$$Register, $op2$$Register);
14208 %}
14209
14210 ins_pipe(icmp_reg_reg);
14211 %}
14212
14213 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14214 %{
14215 match(Set cr (OverflowAddL op1 op2));
14216
14217 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14218 ins_cost(INSN_COST);
14219 ins_encode %{
14220 __ adds(zr, $op1$$Register, $op2$$constant);
14221 %}
14222
14223 ins_pipe(icmp_reg_imm);
14224 %}
14225
14226 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14227 %{
14228 match(Set cr (OverflowSubI op1 op2));
14229
14230 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14231 ins_cost(INSN_COST);
14232 ins_encode %{
14233 __ cmpw($op1$$Register, $op2$$Register);
14234 %}
14235
14236 ins_pipe(icmp_reg_reg);
14237 %}
14238
14239 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14240 %{
14241 match(Set cr (OverflowSubI op1 op2));
14242
14243 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14244 ins_cost(INSN_COST);
14245 ins_encode %{
14246 __ cmpw($op1$$Register, $op2$$constant);
14247 %}
14248
14249 ins_pipe(icmp_reg_imm);
14250 %}
14251
14252 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14253 %{
14254 match(Set cr (OverflowSubL op1 op2));
14255
14256 format %{ "cmp $op1, $op2\t# overflow check long" %}
14257 ins_cost(INSN_COST);
14258 ins_encode %{
14259 __ cmp($op1$$Register, $op2$$Register);
14260 %}
14261
14262 ins_pipe(icmp_reg_reg);
14263 %}
14264
14265 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14266 %{
14267 match(Set cr (OverflowSubL op1 op2));
14268
14269 format %{ "cmp $op1, $op2\t# overflow check long" %}
14270 ins_cost(INSN_COST);
14271 ins_encode %{
14272 __ subs(zr, $op1$$Register, $op2$$constant);
14273 %}
14274
14275 ins_pipe(icmp_reg_imm);
14276 %}
14277
14278 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14279 %{
14280 match(Set cr (OverflowSubI zero op1));
14281
14282 format %{ "cmpw zr, $op1\t# overflow check int" %}
14283 ins_cost(INSN_COST);
14284 ins_encode %{
14285 __ cmpw(zr, $op1$$Register);
14286 %}
14287
14288 ins_pipe(icmp_reg_imm);
14289 %}
14290
14291 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14292 %{
14293 match(Set cr (OverflowSubL zero op1));
14294
14295 format %{ "cmp zr, $op1\t# overflow check long" %}
14296 ins_cost(INSN_COST);
14297 ins_encode %{
14298 __ cmp(zr, $op1$$Register);
14299 %}
14300
14301 ins_pipe(icmp_reg_imm);
14302 %}
14303
14304 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14305 %{
14306 match(Set cr (OverflowMulI op1 op2));
14307
14308 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14309 "cmp rscratch1, rscratch1, sxtw\n\t"
14310 "movw rscratch1, #0x80000000\n\t"
14311 "cselw rscratch1, rscratch1, zr, NE\n\t"
14312 "cmpw rscratch1, #1" %}
14313 ins_cost(5 * INSN_COST);
14314 ins_encode %{
14315 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14316 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14317 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14318 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14319 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14320 %}
14321
14322 ins_pipe(pipe_slow);
14323 %}
14324
14325 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14326 %{
14327 match(If cmp (OverflowMulI op1 op2));
14328 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14329 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14330 effect(USE labl, KILL cr);
14331
14332 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14333 "cmp rscratch1, rscratch1, sxtw\n\t"
14334 "b$cmp $labl" %}
14335 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14336 ins_encode %{
14337 Label* L = $labl$$label;
14338 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14339 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14340 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14341 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14342 %}
14343
14344 ins_pipe(pipe_serial);
14345 %}
14346
14347 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14348 %{
14349 match(Set cr (OverflowMulL op1 op2));
14350
14351 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14352 "smulh rscratch2, $op1, $op2\n\t"
14353 "cmp rscratch2, rscratch1, ASR #63\n\t"
14354 "movw rscratch1, #0x80000000\n\t"
14355 "cselw rscratch1, rscratch1, zr, NE\n\t"
14356 "cmpw rscratch1, #1" %}
14357 ins_cost(6 * INSN_COST);
14358 ins_encode %{
14359 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14360 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14361 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14362 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14363 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14364 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14365 %}
14366
14367 ins_pipe(pipe_slow);
14368 %}
14369
14370 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14371 %{
14372 match(If cmp (OverflowMulL op1 op2));
14373 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14374 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14375 effect(USE labl, KILL cr);
14376
14377 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14378 "smulh rscratch2, $op1, $op2\n\t"
14379 "cmp rscratch2, rscratch1, ASR #63\n\t"
14380 "b$cmp $labl" %}
14381 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14382 ins_encode %{
14383 Label* L = $labl$$label;
14384 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14385 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14386 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14387 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14388 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14389 %}
14390
14391 ins_pipe(pipe_serial);
14392 %}
14393
14394 // ============================================================================
14395 // Compare Instructions
14396
14397 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14398 %{
14399 match(Set cr (CmpI op1 op2));
14400
14401 effect(DEF cr, USE op1, USE op2);
14402
14403 ins_cost(INSN_COST);
14404 format %{ "cmpw $op1, $op2" %}
14405
14406 ins_encode(aarch64_enc_cmpw(op1, op2));
14407
14408 ins_pipe(icmp_reg_reg);
14409 %}
14410
14411 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14412 %{
14413 match(Set cr (CmpI op1 zero));
14414
14415 effect(DEF cr, USE op1);
14416
14417 ins_cost(INSN_COST);
14418 format %{ "cmpw $op1, 0" %}
14419
14420 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14421
14422 ins_pipe(icmp_reg_imm);
14423 %}
14424
14425 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14426 %{
14427 match(Set cr (CmpI op1 op2));
14428
14429 effect(DEF cr, USE op1);
14430
14431 ins_cost(INSN_COST);
14432 format %{ "cmpw $op1, $op2" %}
14433
14434 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14435
14436 ins_pipe(icmp_reg_imm);
14437 %}
14438
14439 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14440 %{
14441 match(Set cr (CmpI op1 op2));
14442
14443 effect(DEF cr, USE op1);
14444
14445 ins_cost(INSN_COST * 2);
14446 format %{ "cmpw $op1, $op2" %}
14447
14448 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14449
14450 ins_pipe(icmp_reg_imm);
14451 %}
14452
14453 // Unsigned compare Instructions; really, same as signed compare
14454 // except it should only be used to feed an If or a CMovI which takes a
14455 // cmpOpU.
14456
14457 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14458 %{
14459 match(Set cr (CmpU op1 op2));
14460
14461 effect(DEF cr, USE op1, USE op2);
14462
14463 ins_cost(INSN_COST);
14464 format %{ "cmpw $op1, $op2\t# unsigned" %}
14465
14466 ins_encode(aarch64_enc_cmpw(op1, op2));
14467
14468 ins_pipe(icmp_reg_reg);
14469 %}
14470
14471 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14472 %{
14473 match(Set cr (CmpU op1 zero));
14474
14475 effect(DEF cr, USE op1);
14476
14477 ins_cost(INSN_COST);
14478 format %{ "cmpw $op1, #0\t# unsigned" %}
14479
14480 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14481
14482 ins_pipe(icmp_reg_imm);
14483 %}
14484
14485 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14486 %{
14487 match(Set cr (CmpU op1 op2));
14488
14489 effect(DEF cr, USE op1);
14490
14491 ins_cost(INSN_COST);
14492 format %{ "cmpw $op1, $op2\t# unsigned" %}
14493
14494 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14495
14496 ins_pipe(icmp_reg_imm);
14497 %}
14498
14499 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14500 %{
14501 match(Set cr (CmpU op1 op2));
14502
14503 effect(DEF cr, USE op1);
14504
14505 ins_cost(INSN_COST * 2);
14506 format %{ "cmpw $op1, $op2\t# unsigned" %}
14507
14508 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14509
14510 ins_pipe(icmp_reg_imm);
14511 %}
14512
14513 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14514 %{
14515 match(Set cr (CmpL op1 op2));
14516
14517 effect(DEF cr, USE op1, USE op2);
14518
14519 ins_cost(INSN_COST);
14520 format %{ "cmp $op1, $op2" %}
14521
14522 ins_encode(aarch64_enc_cmp(op1, op2));
14523
14524 ins_pipe(icmp_reg_reg);
14525 %}
14526
14527 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14528 %{
14529 match(Set cr (CmpL op1 zero));
14530
14531 effect(DEF cr, USE op1);
14532
14533 ins_cost(INSN_COST);
14534 format %{ "tst $op1" %}
14535
14536 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14537
14538 ins_pipe(icmp_reg_imm);
14539 %}
14540
14541 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14542 %{
14543 match(Set cr (CmpL op1 op2));
14544
14545 effect(DEF cr, USE op1);
14546
14547 ins_cost(INSN_COST);
14548 format %{ "cmp $op1, $op2" %}
14549
14550 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14551
14552 ins_pipe(icmp_reg_imm);
14553 %}
14554
14555 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14556 %{
14557 match(Set cr (CmpL op1 op2));
14558
14559 effect(DEF cr, USE op1);
14560
14561 ins_cost(INSN_COST * 2);
14562 format %{ "cmp $op1, $op2" %}
14563
14564 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14565
14566 ins_pipe(icmp_reg_imm);
14567 %}
14568
14569 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14570 %{
14571 match(Set cr (CmpUL op1 op2));
14572
14573 effect(DEF cr, USE op1, USE op2);
14574
14575 ins_cost(INSN_COST);
14576 format %{ "cmp $op1, $op2" %}
14577
14578 ins_encode(aarch64_enc_cmp(op1, op2));
14579
14580 ins_pipe(icmp_reg_reg);
14581 %}
14582
14583 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14584 %{
14585 match(Set cr (CmpUL op1 zero));
14586
14587 effect(DEF cr, USE op1);
14588
14589 ins_cost(INSN_COST);
14590 format %{ "tst $op1" %}
14591
14592 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14593
14594 ins_pipe(icmp_reg_imm);
14595 %}
14596
14597 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14598 %{
14599 match(Set cr (CmpUL op1 op2));
14600
14601 effect(DEF cr, USE op1);
14602
14603 ins_cost(INSN_COST);
14604 format %{ "cmp $op1, $op2" %}
14605
14606 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14607
14608 ins_pipe(icmp_reg_imm);
14609 %}
14610
14611 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14612 %{
14613 match(Set cr (CmpUL op1 op2));
14614
14615 effect(DEF cr, USE op1);
14616
14617 ins_cost(INSN_COST * 2);
14618 format %{ "cmp $op1, $op2" %}
14619
14620 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14621
14622 ins_pipe(icmp_reg_imm);
14623 %}
14624
14625 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14626 %{
14627 match(Set cr (CmpP op1 op2));
14628
14629 effect(DEF cr, USE op1, USE op2);
14630
14631 ins_cost(INSN_COST);
14632 format %{ "cmp $op1, $op2\t // ptr" %}
14633
14634 ins_encode(aarch64_enc_cmpp(op1, op2));
14635
14636 ins_pipe(icmp_reg_reg);
14637 %}
14638
14639 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14640 %{
14641 match(Set cr (CmpN op1 op2));
14642
14643 effect(DEF cr, USE op1, USE op2);
14644
14645 ins_cost(INSN_COST);
14646 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14647
14648 ins_encode(aarch64_enc_cmpn(op1, op2));
14649
14650 ins_pipe(icmp_reg_reg);
14651 %}
14652
14653 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14654 %{
14655 match(Set cr (CmpP op1 zero));
14656
14657 effect(DEF cr, USE op1, USE zero);
14658
14659 ins_cost(INSN_COST);
14660 format %{ "cmp $op1, 0\t // ptr" %}
14661
14662 ins_encode(aarch64_enc_testp(op1));
14663
14664 ins_pipe(icmp_reg_imm);
14665 %}
14666
14667 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14668 %{
14669 match(Set cr (CmpN op1 zero));
14670
14671 effect(DEF cr, USE op1, USE zero);
14672
14673 ins_cost(INSN_COST);
14674 format %{ "cmp $op1, 0\t // compressed ptr" %}
14675
14676 ins_encode(aarch64_enc_testn(op1));
14677
14678 ins_pipe(icmp_reg_imm);
14679 %}
14680
14681 // FP comparisons
14682 //
14683 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14684 // using normal cmpOp. See declaration of rFlagsReg for details.
14685
14686 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14687 %{
14688 match(Set cr (CmpF src1 src2));
14689
14690 ins_cost(3 * INSN_COST);
14691 format %{ "fcmps $src1, $src2" %}
14692
14693 ins_encode %{
14694 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14695 %}
14696
14697 ins_pipe(pipe_class_compare);
14698 %}
14699
14700 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14701 %{
14702 match(Set cr (CmpF src1 src2));
14703
14704 ins_cost(3 * INSN_COST);
14705 format %{ "fcmps $src1, 0.0" %}
14706
14707 ins_encode %{
14708 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14709 %}
14710
14711 ins_pipe(pipe_class_compare);
14712 %}
14713 // FROM HERE
14714
14715 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14716 %{
14717 match(Set cr (CmpD src1 src2));
14718
14719 ins_cost(3 * INSN_COST);
14720 format %{ "fcmpd $src1, $src2" %}
14721
14722 ins_encode %{
14723 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14724 %}
14725
14726 ins_pipe(pipe_class_compare);
14727 %}
14728
14729 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14730 %{
14731 match(Set cr (CmpD src1 src2));
14732
14733 ins_cost(3 * INSN_COST);
14734 format %{ "fcmpd $src1, 0.0" %}
14735
14736 ins_encode %{
14737 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14738 %}
14739
14740 ins_pipe(pipe_class_compare);
14741 %}
14742
14743 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14744 %{
14745 match(Set dst (CmpF3 src1 src2));
14746 effect(KILL cr);
14747
14748 ins_cost(5 * INSN_COST);
14749 format %{ "fcmps $src1, $src2\n\t"
14750 "csinvw($dst, zr, zr, eq\n\t"
14751 "csnegw($dst, $dst, $dst, lt)"
14752 %}
14753
14754 ins_encode %{
14755 Label done;
14756 FloatRegister s1 = as_FloatRegister($src1$$reg);
14757 FloatRegister s2 = as_FloatRegister($src2$$reg);
14758 Register d = as_Register($dst$$reg);
14759 __ fcmps(s1, s2);
14760 // installs 0 if EQ else -1
14761 __ csinvw(d, zr, zr, Assembler::EQ);
14762 // keeps -1 if less or unordered else installs 1
14763 __ csnegw(d, d, d, Assembler::LT);
14764 __ bind(done);
14765 %}
14766
14767 ins_pipe(pipe_class_default);
14768
14769 %}
14770
14771 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14772 %{
14773 match(Set dst (CmpD3 src1 src2));
14774 effect(KILL cr);
14775
14776 ins_cost(5 * INSN_COST);
14777 format %{ "fcmpd $src1, $src2\n\t"
14778 "csinvw($dst, zr, zr, eq\n\t"
14779 "csnegw($dst, $dst, $dst, lt)"
14780 %}
14781
14782 ins_encode %{
14783 Label done;
14784 FloatRegister s1 = as_FloatRegister($src1$$reg);
14785 FloatRegister s2 = as_FloatRegister($src2$$reg);
14786 Register d = as_Register($dst$$reg);
14787 __ fcmpd(s1, s2);
14788 // installs 0 if EQ else -1
14789 __ csinvw(d, zr, zr, Assembler::EQ);
14790 // keeps -1 if less or unordered else installs 1
14791 __ csnegw(d, d, d, Assembler::LT);
14792 __ bind(done);
14793 %}
14794 ins_pipe(pipe_class_default);
14795
14796 %}
14797
14798 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14799 %{
14800 match(Set dst (CmpF3 src1 zero));
14801 effect(KILL cr);
14802
14803 ins_cost(5 * INSN_COST);
14804 format %{ "fcmps $src1, 0.0\n\t"
14805 "csinvw($dst, zr, zr, eq\n\t"
14806 "csnegw($dst, $dst, $dst, lt)"
14807 %}
14808
14809 ins_encode %{
14810 Label done;
14811 FloatRegister s1 = as_FloatRegister($src1$$reg);
14812 Register d = as_Register($dst$$reg);
14813 __ fcmps(s1, 0.0);
14814 // installs 0 if EQ else -1
14815 __ csinvw(d, zr, zr, Assembler::EQ);
14816 // keeps -1 if less or unordered else installs 1
14817 __ csnegw(d, d, d, Assembler::LT);
14818 __ bind(done);
14819 %}
14820
14821 ins_pipe(pipe_class_default);
14822
14823 %}
14824
14825 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14826 %{
14827 match(Set dst (CmpD3 src1 zero));
14828 effect(KILL cr);
14829
14830 ins_cost(5 * INSN_COST);
14831 format %{ "fcmpd $src1, 0.0\n\t"
14832 "csinvw($dst, zr, zr, eq\n\t"
14833 "csnegw($dst, $dst, $dst, lt)"
14834 %}
14835
14836 ins_encode %{
14837 Label done;
14838 FloatRegister s1 = as_FloatRegister($src1$$reg);
14839 Register d = as_Register($dst$$reg);
14840 __ fcmpd(s1, 0.0);
14841 // installs 0 if EQ else -1
14842 __ csinvw(d, zr, zr, Assembler::EQ);
14843 // keeps -1 if less or unordered else installs 1
14844 __ csnegw(d, d, d, Assembler::LT);
14845 __ bind(done);
14846 %}
14847 ins_pipe(pipe_class_default);
14848
14849 %}
14850
14851 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14852 %{
14853 match(Set dst (CmpLTMask p q));
14854 effect(KILL cr);
14855
14856 ins_cost(3 * INSN_COST);
14857
14858 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14859 "csetw $dst, lt\n\t"
14860 "subw $dst, zr, $dst"
14861 %}
14862
14863 ins_encode %{
14864 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14865 __ csetw(as_Register($dst$$reg), Assembler::LT);
14866 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14867 %}
14868
14869 ins_pipe(ialu_reg_reg);
14870 %}
14871
14872 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14873 %{
14874 match(Set dst (CmpLTMask src zero));
14875 effect(KILL cr);
14876
14877 ins_cost(INSN_COST);
14878
14879 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14880
14881 ins_encode %{
14882 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14883 %}
14884
14885 ins_pipe(ialu_reg_shift);
14886 %}
14887
14888 // ============================================================================
14889 // Max and Min
14890
14891 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14892
14893 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14894 %{
14895 effect(DEF cr, USE src);
14896 ins_cost(INSN_COST);
14897 format %{ "cmpw $src, 0" %}
14898
14899 ins_encode %{
14900 __ cmpw($src$$Register, 0);
14901 %}
14902 ins_pipe(icmp_reg_imm);
14903 %}
14904
14905 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14906 %{
14907 match(Set dst (MinI src1 src2));
14908 ins_cost(INSN_COST * 3);
14909
14910 expand %{
14911 rFlagsReg cr;
14912 compI_reg_reg(cr, src1, src2);
14913 cmovI_reg_reg_lt(dst, src1, src2, cr);
14914 %}
14915 %}
14916
14917 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14918 %{
14919 match(Set dst (MaxI src1 src2));
14920 ins_cost(INSN_COST * 3);
14921
14922 expand %{
14923 rFlagsReg cr;
14924 compI_reg_reg(cr, src1, src2);
14925 cmovI_reg_reg_gt(dst, src1, src2, cr);
14926 %}
14927 %}
14928
14929
14930 // ============================================================================
14931 // Branch Instructions
14932
14933 // Direct Branch.
14934 instruct branch(label lbl)
14935 %{
14936 match(Goto);
14937
14938 effect(USE lbl);
14939
14940 ins_cost(BRANCH_COST);
14941 format %{ "b $lbl" %}
14942
14943 ins_encode(aarch64_enc_b(lbl));
14944
14945 ins_pipe(pipe_branch);
14946 %}
14947
14948 // Conditional Near Branch
14949 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14950 %{
14951 // Same match rule as `branchConFar'.
14952 match(If cmp cr);
14953
14954 effect(USE lbl);
14955
14956 ins_cost(BRANCH_COST);
14957 // If set to 1 this indicates that the current instruction is a
14958 // short variant of a long branch. This avoids using this
14959 // instruction in first-pass matching. It will then only be used in
14960 // the `Shorten_branches' pass.
14961 // ins_short_branch(1);
14962 format %{ "b$cmp $lbl" %}
14963
14964 ins_encode(aarch64_enc_br_con(cmp, lbl));
14965
14966 ins_pipe(pipe_branch_cond);
14967 %}
14968
14969 // Conditional Near Branch Unsigned
14970 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14971 %{
14972 // Same match rule as `branchConFar'.
14973 match(If cmp cr);
14974
14975 effect(USE lbl);
14976
14977 ins_cost(BRANCH_COST);
14978 // If set to 1 this indicates that the current instruction is a
14979 // short variant of a long branch. This avoids using this
14980 // instruction in first-pass matching. It will then only be used in
14981 // the `Shorten_branches' pass.
14982 // ins_short_branch(1);
14983 format %{ "b$cmp $lbl\t# unsigned" %}
14984
14985 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14986
14987 ins_pipe(pipe_branch_cond);
14988 %}
14989
14990 // Make use of CBZ and CBNZ. These instructions, as well as being
14991 // shorter than (cmp; branch), have the additional benefit of not
14992 // killing the flags.
14993
14994 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14995 match(If cmp (CmpI op1 op2));
14996 effect(USE labl);
14997
14998 ins_cost(BRANCH_COST);
14999 format %{ "cbw$cmp $op1, $labl" %}
15000 ins_encode %{
15001 Label* L = $labl$$label;
15002 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15003 if (cond == Assembler::EQ)
15004 __ cbzw($op1$$Register, *L);
15005 else
15006 __ cbnzw($op1$$Register, *L);
15007 %}
15008 ins_pipe(pipe_cmp_branch);
15009 %}
15010
15011 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15012 match(If cmp (CmpL op1 op2));
15013 effect(USE labl);
15014
15015 ins_cost(BRANCH_COST);
15016 format %{ "cb$cmp $op1, $labl" %}
15017 ins_encode %{
15018 Label* L = $labl$$label;
15019 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15020 if (cond == Assembler::EQ)
15021 __ cbz($op1$$Register, *L);
15022 else
15023 __ cbnz($op1$$Register, *L);
15024 %}
15025 ins_pipe(pipe_cmp_branch);
15026 %}
15027
15028 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15029 match(If cmp (CmpP op1 op2));
15030 effect(USE labl);
15031
15032 ins_cost(BRANCH_COST);
15033 format %{ "cb$cmp $op1, $labl" %}
15034 ins_encode %{
15035 Label* L = $labl$$label;
15036 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15037 if (cond == Assembler::EQ)
15038 __ cbz($op1$$Register, *L);
15039 else
15040 __ cbnz($op1$$Register, *L);
15041 %}
15042 ins_pipe(pipe_cmp_branch);
15043 %}
15044
15045 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15046 match(If cmp (CmpN op1 op2));
15047 effect(USE labl);
15048
15049 ins_cost(BRANCH_COST);
15050 format %{ "cbw$cmp $op1, $labl" %}
15051 ins_encode %{
15052 Label* L = $labl$$label;
15053 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15054 if (cond == Assembler::EQ)
15055 __ cbzw($op1$$Register, *L);
15056 else
15057 __ cbnzw($op1$$Register, *L);
15058 %}
15059 ins_pipe(pipe_cmp_branch);
15060 %}
15061
15062 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15063 match(If cmp (CmpP (DecodeN oop) zero));
15064 effect(USE labl);
15065
15066 ins_cost(BRANCH_COST);
15067 format %{ "cb$cmp $oop, $labl" %}
15068 ins_encode %{
15069 Label* L = $labl$$label;
15070 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15071 if (cond == Assembler::EQ)
15072 __ cbzw($oop$$Register, *L);
15073 else
15074 __ cbnzw($oop$$Register, *L);
15075 %}
15076 ins_pipe(pipe_cmp_branch);
15077 %}
15078
15079 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15080 match(If cmp (CmpU op1 op2));
15081 effect(USE labl);
15082
15083 ins_cost(BRANCH_COST);
15084 format %{ "cbw$cmp $op1, $labl" %}
15085 ins_encode %{
15086 Label* L = $labl$$label;
15087 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15088 if (cond == Assembler::EQ || cond == Assembler::LS) {
15089 __ cbzw($op1$$Register, *L);
15090 } else {
15091 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15092 __ cbnzw($op1$$Register, *L);
15093 }
15094 %}
15095 ins_pipe(pipe_cmp_branch);
15096 %}
15097
15098 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15099 match(If cmp (CmpUL op1 op2));
15100 effect(USE labl);
15101
15102 ins_cost(BRANCH_COST);
15103 format %{ "cb$cmp $op1, $labl" %}
15104 ins_encode %{
15105 Label* L = $labl$$label;
15106 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15107 if (cond == Assembler::EQ || cond == Assembler::LS) {
15108 __ cbz($op1$$Register, *L);
15109 } else {
15110 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15111 __ cbnz($op1$$Register, *L);
15112 }
15113 %}
15114 ins_pipe(pipe_cmp_branch);
15115 %}
15116
15117 // Test bit and Branch
15118
15119 // Patterns for short (< 32KiB) variants
15120 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15121 match(If cmp (CmpL op1 op2));
15122 effect(USE labl);
15123
15124 ins_cost(BRANCH_COST);
15125 format %{ "cb$cmp $op1, $labl # long" %}
15126 ins_encode %{
15127 Label* L = $labl$$label;
15128 Assembler::Condition cond =
15129 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15130 __ tbr(cond, $op1$$Register, 63, *L);
15131 %}
15132 ins_pipe(pipe_cmp_branch);
15133 ins_short_branch(1);
15134 %}
15135
15136 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15137 match(If cmp (CmpI op1 op2));
15138 effect(USE labl);
15139
15140 ins_cost(BRANCH_COST);
15141 format %{ "cb$cmp $op1, $labl # int" %}
15142 ins_encode %{
15143 Label* L = $labl$$label;
15144 Assembler::Condition cond =
15145 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15146 __ tbr(cond, $op1$$Register, 31, *L);
15147 %}
15148 ins_pipe(pipe_cmp_branch);
15149 ins_short_branch(1);
15150 %}
15151
15152 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15153 match(If cmp (CmpL (AndL op1 op2) op3));
15154 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15155 effect(USE labl);
15156
15157 ins_cost(BRANCH_COST);
15158 format %{ "tb$cmp $op1, $op2, $labl" %}
15159 ins_encode %{
15160 Label* L = $labl$$label;
15161 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15162 int bit = exact_log2_long($op2$$constant);
15163 __ tbr(cond, $op1$$Register, bit, *L);
15164 %}
15165 ins_pipe(pipe_cmp_branch);
15166 ins_short_branch(1);
15167 %}
15168
15169 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15170 match(If cmp (CmpI (AndI op1 op2) op3));
15171 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15172 effect(USE labl);
15173
15174 ins_cost(BRANCH_COST);
15175 format %{ "tb$cmp $op1, $op2, $labl" %}
15176 ins_encode %{
15177 Label* L = $labl$$label;
15178 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15179 int bit = exact_log2((juint)$op2$$constant);
15180 __ tbr(cond, $op1$$Register, bit, *L);
15181 %}
15182 ins_pipe(pipe_cmp_branch);
15183 ins_short_branch(1);
15184 %}
15185
15186 // And far variants
15187 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15188 match(If cmp (CmpL op1 op2));
15189 effect(USE labl);
15190
15191 ins_cost(BRANCH_COST);
15192 format %{ "cb$cmp $op1, $labl # long" %}
15193 ins_encode %{
15194 Label* L = $labl$$label;
15195 Assembler::Condition cond =
15196 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15197 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15198 %}
15199 ins_pipe(pipe_cmp_branch);
15200 %}
15201
15202 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15203 match(If cmp (CmpI op1 op2));
15204 effect(USE labl);
15205
15206 ins_cost(BRANCH_COST);
15207 format %{ "cb$cmp $op1, $labl # int" %}
15208 ins_encode %{
15209 Label* L = $labl$$label;
15210 Assembler::Condition cond =
15211 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15212 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15213 %}
15214 ins_pipe(pipe_cmp_branch);
15215 %}
15216
15217 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15218 match(If cmp (CmpL (AndL op1 op2) op3));
15219 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15220 effect(USE labl);
15221
15222 ins_cost(BRANCH_COST);
15223 format %{ "tb$cmp $op1, $op2, $labl" %}
15224 ins_encode %{
15225 Label* L = $labl$$label;
15226 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15227 int bit = exact_log2_long($op2$$constant);
15228 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15229 %}
15230 ins_pipe(pipe_cmp_branch);
15231 %}
15232
15233 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15234 match(If cmp (CmpI (AndI op1 op2) op3));
15235 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15236 effect(USE labl);
15237
15238 ins_cost(BRANCH_COST);
15239 format %{ "tb$cmp $op1, $op2, $labl" %}
15240 ins_encode %{
15241 Label* L = $labl$$label;
15242 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15243 int bit = exact_log2((juint)$op2$$constant);
15244 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15245 %}
15246 ins_pipe(pipe_cmp_branch);
15247 %}
15248
15249 // Test bits
15250
15251 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15252 match(Set cr (CmpL (AndL op1 op2) op3));
15253 predicate(Assembler::operand_valid_for_logical_immediate
15254 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15255
15256 ins_cost(INSN_COST);
15257 format %{ "tst $op1, $op2 # long" %}
15258 ins_encode %{
15259 __ tst($op1$$Register, $op2$$constant);
15260 %}
15261 ins_pipe(ialu_reg_reg);
15262 %}
15263
15264 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15265 match(Set cr (CmpI (AndI op1 op2) op3));
15266 predicate(Assembler::operand_valid_for_logical_immediate
15267 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15268
15269 ins_cost(INSN_COST);
15270 format %{ "tst $op1, $op2 # int" %}
15271 ins_encode %{
15272 __ tstw($op1$$Register, $op2$$constant);
15273 %}
15274 ins_pipe(ialu_reg_reg);
15275 %}
15276
15277 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15278 match(Set cr (CmpL (AndL op1 op2) op3));
15279
15280 ins_cost(INSN_COST);
15281 format %{ "tst $op1, $op2 # long" %}
15282 ins_encode %{
15283 __ tst($op1$$Register, $op2$$Register);
15284 %}
15285 ins_pipe(ialu_reg_reg);
15286 %}
15287
15288 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15289 match(Set cr (CmpI (AndI op1 op2) op3));
15290
15291 ins_cost(INSN_COST);
15292 format %{ "tstw $op1, $op2 # int" %}
15293 ins_encode %{
15294 __ tstw($op1$$Register, $op2$$Register);
15295 %}
15296 ins_pipe(ialu_reg_reg);
15297 %}
15298
15299
15300 // Conditional Far Branch
15301 // Conditional Far Branch Unsigned
15302 // TODO: fixme
15303
15304 // counted loop end branch near
15305 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15306 %{
15307 match(CountedLoopEnd cmp cr);
15308
15309 effect(USE lbl);
15310
15311 ins_cost(BRANCH_COST);
15312 // short variant.
15313 // ins_short_branch(1);
15314 format %{ "b$cmp $lbl \t// counted loop end" %}
15315
15316 ins_encode(aarch64_enc_br_con(cmp, lbl));
15317
15318 ins_pipe(pipe_branch);
15319 %}
15320
15321 // counted loop end branch far
15322 // TODO: fixme
15323
15324 // ============================================================================
15325 // inlined locking and unlocking
15326
15327 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15328 %{
15329 match(Set cr (FastLock object box));
15330 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15331
15332 ins_cost(5 * INSN_COST);
15333 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15334
15335 ins_encode %{
15336 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15337 %}
15338
15339 ins_pipe(pipe_serial);
15340 %}
15341
15342 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15343 %{
15344 match(Set cr (FastUnlock object box));
15345 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15346
15347 ins_cost(5 * INSN_COST);
15348 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15349
15350 ins_encode %{
15351 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15352 %}
15353
15354 ins_pipe(pipe_serial);
15355 %}
15356
15357 // ============================================================================
15358 // Safepoint Instructions
15359
15360 // TODO
15361 // provide a near and far version of this code
15362
15363 instruct safePoint(rFlagsReg cr, iRegP poll)
15364 %{
15365 match(SafePoint poll);
15366 effect(KILL cr);
15367
15368 format %{
15369 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15370 %}
15371 ins_encode %{
15372 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15373 %}
15374 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15375 %}
15376
15377
15378 // ============================================================================
15379 // Procedure Call/Return Instructions
15380
15381 // Call Java Static Instruction
15382
15383 instruct CallStaticJavaDirect(method meth)
15384 %{
15385 match(CallStaticJava);
15386
15387 effect(USE meth);
15388
15389 ins_cost(CALL_COST);
15390
15391 format %{ "call,static $meth \t// ==> " %}
15392
15393 ins_encode(aarch64_enc_java_static_call(meth),
15394 aarch64_enc_call_epilog);
15395
15396 ins_pipe(pipe_class_call);
15397 %}
15398
15399 // TO HERE
15400
15401 // Call Java Dynamic Instruction
15402 instruct CallDynamicJavaDirect(method meth)
15403 %{
15404 match(CallDynamicJava);
15405
15406 effect(USE meth);
15407
15408 ins_cost(CALL_COST);
15409
15410 format %{ "CALL,dynamic $meth \t// ==> " %}
15411
15412 ins_encode(aarch64_enc_java_dynamic_call(meth),
15413 aarch64_enc_call_epilog);
15414
15415 ins_pipe(pipe_class_call);
15416 %}
15417
15418 // Call Runtime Instruction
15419
15420 instruct CallRuntimeDirect(method meth)
15421 %{
15422 match(CallRuntime);
15423
15424 effect(USE meth);
15425
15426 ins_cost(CALL_COST);
15427
15428 format %{ "CALL, runtime $meth" %}
15429
15430 ins_encode( aarch64_enc_java_to_runtime(meth) );
15431
15432 ins_pipe(pipe_class_call);
15433 %}
15434
15435 // Call Runtime Instruction
15436
15437 instruct CallLeafDirect(method meth)
15438 %{
15439 match(CallLeaf);
15440
15441 effect(USE meth);
15442
15443 ins_cost(CALL_COST);
15444
15445 format %{ "CALL, runtime leaf $meth" %}
15446
15447 ins_encode( aarch64_enc_java_to_runtime(meth) );
15448
15449 ins_pipe(pipe_class_call);
15450 %}
15451
15452 // Call Runtime Instruction without safepoint and with vector arguments
15453 instruct CallLeafDirectVector(method meth)
15454 %{
15455 match(CallLeafVector);
15456
15457 effect(USE meth);
15458
15459 ins_cost(CALL_COST);
15460
15461 format %{ "CALL, runtime leaf vector $meth" %}
15462
15463 ins_encode(aarch64_enc_java_to_runtime(meth));
15464
15465 ins_pipe(pipe_class_call);
15466 %}
15467
15468 // Call Runtime Instruction
15469
15470 instruct CallLeafNoFPDirect(method meth)
15471 %{
15472 match(CallLeafNoFP);
15473
15474 effect(USE meth);
15475
15476 ins_cost(CALL_COST);
15477
15478 format %{ "CALL, runtime leaf nofp $meth" %}
15479
15480 ins_encode( aarch64_enc_java_to_runtime(meth) );
15481
15482 ins_pipe(pipe_class_call);
15483 %}
15484
15485 // Tail Call; Jump from runtime stub to Java code.
15486 // Also known as an 'interprocedural jump'.
15487 // Target of jump will eventually return to caller.
15488 // TailJump below removes the return address.
15489 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15490 // emitted just above the TailCall which has reset rfp to the caller state.
15491 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15492 %{
15493 match(TailCall jump_target method_ptr);
15494
15495 ins_cost(CALL_COST);
15496
15497 format %{ "br $jump_target\t# $method_ptr holds method" %}
15498
15499 ins_encode(aarch64_enc_tail_call(jump_target));
15500
15501 ins_pipe(pipe_class_call);
15502 %}
15503
15504 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15505 %{
15506 match(TailJump jump_target ex_oop);
15507
15508 ins_cost(CALL_COST);
15509
15510 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15511
15512 ins_encode(aarch64_enc_tail_jmp(jump_target));
15513
15514 ins_pipe(pipe_class_call);
15515 %}
15516
15517 // Forward exception.
15518 instruct ForwardExceptionjmp()
15519 %{
15520 match(ForwardException);
15521 ins_cost(CALL_COST);
15522
15523 format %{ "b forward_exception_stub" %}
15524 ins_encode %{
15525 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15526 %}
15527 ins_pipe(pipe_class_call);
15528 %}
15529
15530 // Create exception oop: created by stack-crawling runtime code.
15531 // Created exception is now available to this handler, and is setup
15532 // just prior to jumping to this handler. No code emitted.
15533 // TODO check
15534 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15535 instruct CreateException(iRegP_R0 ex_oop)
15536 %{
15537 match(Set ex_oop (CreateEx));
15538
15539 format %{ " -- \t// exception oop; no code emitted" %}
15540
15541 size(0);
15542
15543 ins_encode( /*empty*/ );
15544
15545 ins_pipe(pipe_class_empty);
15546 %}
15547
15548 // Rethrow exception: The exception oop will come in the first
15549 // argument position. Then JUMP (not call) to the rethrow stub code.
15550 instruct RethrowException() %{
15551 match(Rethrow);
15552 ins_cost(CALL_COST);
15553
15554 format %{ "b rethrow_stub" %}
15555
15556 ins_encode( aarch64_enc_rethrow() );
15557
15558 ins_pipe(pipe_class_call);
15559 %}
15560
15561
15562 // Return Instruction
15563 // epilog node loads ret address into lr as part of frame pop
15564 instruct Ret()
15565 %{
15566 match(Return);
15567
15568 format %{ "ret\t// return register" %}
15569
15570 ins_encode( aarch64_enc_ret() );
15571
15572 ins_pipe(pipe_branch);
15573 %}
15574
15575 // Die now.
15576 instruct ShouldNotReachHere() %{
15577 match(Halt);
15578
15579 ins_cost(CALL_COST);
15580 format %{ "ShouldNotReachHere" %}
15581
15582 ins_encode %{
15583 if (is_reachable()) {
15584 const char* str = __ code_string(_halt_reason);
15585 __ stop(str);
15586 }
15587 %}
15588
15589 ins_pipe(pipe_class_default);
15590 %}
15591
15592 // ============================================================================
15593 // Partial Subtype Check
15594 //
15595 // superklass array for an instance of the superklass. Set a hidden
15596 // internal cache on a hit (cache is checked with exposed code in
15597 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15598 // encoding ALSO sets flags.
15599
15600 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15601 %{
15602 match(Set result (PartialSubtypeCheck sub super));
15603 predicate(!UseSecondarySupersTable);
15604 effect(KILL cr, KILL temp);
15605
15606 ins_cost(20 * INSN_COST); // slightly larger than the next version
15607 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15608
15609 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15610
15611 opcode(0x1); // Force zero of result reg on hit
15612
15613 ins_pipe(pipe_class_memory);
15614 %}
15615
15616 // Two versions of partialSubtypeCheck, both used when we need to
15617 // search for a super class in the secondary supers array. The first
15618 // is used when we don't know _a priori_ the class being searched
15619 // for. The second, far more common, is used when we do know: this is
15620 // used for instanceof, checkcast, and any case where C2 can determine
15621 // it by constant propagation.
15622
15623 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15624 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15625 rFlagsReg cr)
15626 %{
15627 match(Set result (PartialSubtypeCheck sub super));
15628 predicate(UseSecondarySupersTable);
15629 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15630
15631 ins_cost(10 * INSN_COST); // slightly larger than the next version
15632 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15633
15634 ins_encode %{
15635 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15636 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15637 $vtemp$$FloatRegister,
15638 $result$$Register, /*L_success*/nullptr);
15639 %}
15640
15641 ins_pipe(pipe_class_memory);
15642 %}
15643
15644 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15645 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15646 rFlagsReg cr)
15647 %{
15648 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15649 predicate(UseSecondarySupersTable);
15650 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15651
15652 ins_cost(5 * INSN_COST); // smaller than the next version
15653 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15654
15655 ins_encode %{
15656 bool success = false;
15657 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15658 if (InlineSecondarySupersTest) {
15659 success =
15660 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15661 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15662 $vtemp$$FloatRegister,
15663 $result$$Register,
15664 super_klass_slot);
15665 } else {
15666 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15667 success = (call != nullptr);
15668 }
15669 if (!success) {
15670 ciEnv::current()->record_failure("CodeCache is full");
15671 return;
15672 }
15673 %}
15674
15675 ins_pipe(pipe_class_memory);
15676 %}
15677
15678 // Intrisics for String.compareTo()
15679
15680 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15681 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15682 %{
15683 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15684 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15685 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15686
15687 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15688 ins_encode %{
15689 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15690 __ string_compare($str1$$Register, $str2$$Register,
15691 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15692 $tmp1$$Register, $tmp2$$Register,
15693 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15694 %}
15695 ins_pipe(pipe_class_memory);
15696 %}
15697
15698 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15699 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15700 %{
15701 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15702 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15703 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15704
15705 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15706 ins_encode %{
15707 __ string_compare($str1$$Register, $str2$$Register,
15708 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15709 $tmp1$$Register, $tmp2$$Register,
15710 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15711 %}
15712 ins_pipe(pipe_class_memory);
15713 %}
15714
15715 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15716 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15717 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15718 %{
15719 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15720 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15721 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15722 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15723
15724 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15725 ins_encode %{
15726 __ string_compare($str1$$Register, $str2$$Register,
15727 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15728 $tmp1$$Register, $tmp2$$Register,
15729 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15730 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15731 %}
15732 ins_pipe(pipe_class_memory);
15733 %}
15734
15735 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15736 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15737 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15738 %{
15739 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15740 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15741 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15742 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15743
15744 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15745 ins_encode %{
15746 __ string_compare($str1$$Register, $str2$$Register,
15747 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15748 $tmp1$$Register, $tmp2$$Register,
15749 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15750 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15751 %}
15752 ins_pipe(pipe_class_memory);
15753 %}
15754
15755 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15756 // these string_compare variants as NEON register type for convenience so that the prototype of
15757 // string_compare can be shared with all variants.
15758
15759 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15760 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15761 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15762 pRegGov_P1 pgtmp2, rFlagsReg cr)
15763 %{
15764 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15765 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15766 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15767 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15768
15769 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15770 ins_encode %{
15771 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15772 __ string_compare($str1$$Register, $str2$$Register,
15773 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15774 $tmp1$$Register, $tmp2$$Register,
15775 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15776 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15777 StrIntrinsicNode::LL);
15778 %}
15779 ins_pipe(pipe_class_memory);
15780 %}
15781
15782 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15783 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15784 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15785 pRegGov_P1 pgtmp2, rFlagsReg cr)
15786 %{
15787 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15788 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15789 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15790 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15791
15792 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15793 ins_encode %{
15794 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15795 __ string_compare($str1$$Register, $str2$$Register,
15796 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15797 $tmp1$$Register, $tmp2$$Register,
15798 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15799 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15800 StrIntrinsicNode::LU);
15801 %}
15802 ins_pipe(pipe_class_memory);
15803 %}
15804
15805 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15806 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15807 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15808 pRegGov_P1 pgtmp2, rFlagsReg cr)
15809 %{
15810 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15811 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15812 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15813 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15814
15815 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15816 ins_encode %{
15817 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15818 __ string_compare($str1$$Register, $str2$$Register,
15819 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15820 $tmp1$$Register, $tmp2$$Register,
15821 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15822 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15823 StrIntrinsicNode::UL);
15824 %}
15825 ins_pipe(pipe_class_memory);
15826 %}
15827
15828 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15829 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15830 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15831 pRegGov_P1 pgtmp2, rFlagsReg cr)
15832 %{
15833 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15834 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15835 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15836 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15837
15838 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15839 ins_encode %{
15840 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15841 __ string_compare($str1$$Register, $str2$$Register,
15842 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15843 $tmp1$$Register, $tmp2$$Register,
15844 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15845 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15846 StrIntrinsicNode::UU);
15847 %}
15848 ins_pipe(pipe_class_memory);
15849 %}
15850
15851 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15852 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15853 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15854 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15855 %{
15856 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15857 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15858 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15859 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15860 TEMP vtmp0, TEMP vtmp1, KILL cr);
15861 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15862 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15863
15864 ins_encode %{
15865 __ string_indexof($str1$$Register, $str2$$Register,
15866 $cnt1$$Register, $cnt2$$Register,
15867 $tmp1$$Register, $tmp2$$Register,
15868 $tmp3$$Register, $tmp4$$Register,
15869 $tmp5$$Register, $tmp6$$Register,
15870 -1, $result$$Register, StrIntrinsicNode::UU);
15871 %}
15872 ins_pipe(pipe_class_memory);
15873 %}
15874
15875 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15876 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15877 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15878 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15879 %{
15880 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15881 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15882 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15883 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15884 TEMP vtmp0, TEMP vtmp1, KILL cr);
15885 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15886 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15887
15888 ins_encode %{
15889 __ string_indexof($str1$$Register, $str2$$Register,
15890 $cnt1$$Register, $cnt2$$Register,
15891 $tmp1$$Register, $tmp2$$Register,
15892 $tmp3$$Register, $tmp4$$Register,
15893 $tmp5$$Register, $tmp6$$Register,
15894 -1, $result$$Register, StrIntrinsicNode::LL);
15895 %}
15896 ins_pipe(pipe_class_memory);
15897 %}
15898
15899 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15900 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15901 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15902 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15903 %{
15904 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15905 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15906 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15907 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15908 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15909 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15910 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15911
15912 ins_encode %{
15913 __ string_indexof($str1$$Register, $str2$$Register,
15914 $cnt1$$Register, $cnt2$$Register,
15915 $tmp1$$Register, $tmp2$$Register,
15916 $tmp3$$Register, $tmp4$$Register,
15917 $tmp5$$Register, $tmp6$$Register,
15918 -1, $result$$Register, StrIntrinsicNode::UL);
15919 %}
15920 ins_pipe(pipe_class_memory);
15921 %}
15922
15923 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15924 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15925 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15926 %{
15927 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15928 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15929 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15930 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15931 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15932 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15933
15934 ins_encode %{
15935 int icnt2 = (int)$int_cnt2$$constant;
15936 __ string_indexof($str1$$Register, $str2$$Register,
15937 $cnt1$$Register, zr,
15938 $tmp1$$Register, $tmp2$$Register,
15939 $tmp3$$Register, $tmp4$$Register, zr, zr,
15940 icnt2, $result$$Register, StrIntrinsicNode::UU);
15941 %}
15942 ins_pipe(pipe_class_memory);
15943 %}
15944
15945 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15946 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15947 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15948 %{
15949 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15950 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15951 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15952 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15953 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15954 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15955
15956 ins_encode %{
15957 int icnt2 = (int)$int_cnt2$$constant;
15958 __ string_indexof($str1$$Register, $str2$$Register,
15959 $cnt1$$Register, zr,
15960 $tmp1$$Register, $tmp2$$Register,
15961 $tmp3$$Register, $tmp4$$Register, zr, zr,
15962 icnt2, $result$$Register, StrIntrinsicNode::LL);
15963 %}
15964 ins_pipe(pipe_class_memory);
15965 %}
15966
15967 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15968 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15969 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15970 %{
15971 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15972 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15973 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15974 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15975 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15976 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15977
15978 ins_encode %{
15979 int icnt2 = (int)$int_cnt2$$constant;
15980 __ string_indexof($str1$$Register, $str2$$Register,
15981 $cnt1$$Register, zr,
15982 $tmp1$$Register, $tmp2$$Register,
15983 $tmp3$$Register, $tmp4$$Register, zr, zr,
15984 icnt2, $result$$Register, StrIntrinsicNode::UL);
15985 %}
15986 ins_pipe(pipe_class_memory);
15987 %}
15988
15989 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15990 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15991 iRegINoSp tmp3, rFlagsReg cr)
15992 %{
15993 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15994 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15995 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15996 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15997
15998 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15999
16000 ins_encode %{
16001 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16002 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16003 $tmp3$$Register);
16004 %}
16005 ins_pipe(pipe_class_memory);
16006 %}
16007
16008 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16009 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16010 iRegINoSp tmp3, rFlagsReg cr)
16011 %{
16012 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16013 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16014 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16015 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16016
16017 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16018
16019 ins_encode %{
16020 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16021 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16022 $tmp3$$Register);
16023 %}
16024 ins_pipe(pipe_class_memory);
16025 %}
16026
16027 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16028 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16029 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16030 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16031 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16032 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16033 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16034 ins_encode %{
16035 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16036 $result$$Register, $ztmp1$$FloatRegister,
16037 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16038 $ptmp$$PRegister, true /* isL */);
16039 %}
16040 ins_pipe(pipe_class_memory);
16041 %}
16042
16043 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16044 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16045 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16046 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16047 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16048 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16049 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16050 ins_encode %{
16051 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16052 $result$$Register, $ztmp1$$FloatRegister,
16053 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16054 $ptmp$$PRegister, false /* isL */);
16055 %}
16056 ins_pipe(pipe_class_memory);
16057 %}
16058
16059 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16060 iRegI_R0 result, rFlagsReg cr)
16061 %{
16062 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16063 match(Set result (StrEquals (Binary str1 str2) cnt));
16064 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16065
16066 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16067 ins_encode %{
16068 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16069 __ string_equals($str1$$Register, $str2$$Register,
16070 $result$$Register, $cnt$$Register);
16071 %}
16072 ins_pipe(pipe_class_memory);
16073 %}
16074
16075 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16076 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16077 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16078 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16079 iRegP_R10 tmp, rFlagsReg cr)
16080 %{
16081 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16082 match(Set result (AryEq ary1 ary2));
16083 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16084 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16085 TEMP vtmp6, TEMP vtmp7, KILL cr);
16086
16087 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16088 ins_encode %{
16089 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16090 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16091 $result$$Register, $tmp$$Register, 1);
16092 if (tpc == nullptr) {
16093 ciEnv::current()->record_failure("CodeCache is full");
16094 return;
16095 }
16096 %}
16097 ins_pipe(pipe_class_memory);
16098 %}
16099
16100 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16101 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16102 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16103 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16104 iRegP_R10 tmp, rFlagsReg cr)
16105 %{
16106 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16107 match(Set result (AryEq ary1 ary2));
16108 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16109 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16110 TEMP vtmp6, TEMP vtmp7, KILL cr);
16111
16112 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16113 ins_encode %{
16114 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16115 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16116 $result$$Register, $tmp$$Register, 2);
16117 if (tpc == nullptr) {
16118 ciEnv::current()->record_failure("CodeCache is full");
16119 return;
16120 }
16121 %}
16122 ins_pipe(pipe_class_memory);
16123 %}
16124
16125 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16126 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16127 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16128 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16129 %{
16130 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16131 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16132 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16133
16134 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16135 ins_encode %{
16136 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16137 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16138 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16139 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16140 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16141 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16142 (BasicType)$basic_type$$constant);
16143 if (tpc == nullptr) {
16144 ciEnv::current()->record_failure("CodeCache is full");
16145 return;
16146 }
16147 %}
16148 ins_pipe(pipe_class_memory);
16149 %}
16150
16151 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16152 %{
16153 match(Set result (CountPositives ary1 len));
16154 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16155 format %{ "count positives byte[] $ary1,$len -> $result" %}
16156 ins_encode %{
16157 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16158 if (tpc == nullptr) {
16159 ciEnv::current()->record_failure("CodeCache is full");
16160 return;
16161 }
16162 %}
16163 ins_pipe( pipe_slow );
16164 %}
16165
16166 // fast char[] to byte[] compression
16167 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16168 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16169 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16170 iRegI_R0 result, rFlagsReg cr)
16171 %{
16172 match(Set result (StrCompressedCopy src (Binary dst len)));
16173 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16174 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16175
16176 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16177 ins_encode %{
16178 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16179 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16180 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16181 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16182 %}
16183 ins_pipe(pipe_slow);
16184 %}
16185
16186 // fast byte[] to char[] inflation
16187 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16188 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16189 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16190 %{
16191 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16192 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16193 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16194 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16195
16196 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16197 ins_encode %{
16198 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16199 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16200 $vtmp2$$FloatRegister, $tmp$$Register);
16201 if (tpc == nullptr) {
16202 ciEnv::current()->record_failure("CodeCache is full");
16203 return;
16204 }
16205 %}
16206 ins_pipe(pipe_class_memory);
16207 %}
16208
16209 // encode char[] to byte[] in ISO_8859_1
16210 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16211 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16212 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16213 iRegI_R0 result, rFlagsReg cr)
16214 %{
16215 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16216 match(Set result (EncodeISOArray src (Binary dst len)));
16217 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16218 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16219
16220 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16221 ins_encode %{
16222 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16223 $result$$Register, false,
16224 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16225 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16226 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16227 %}
16228 ins_pipe(pipe_class_memory);
16229 %}
16230
16231 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16232 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16233 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16234 iRegI_R0 result, rFlagsReg cr)
16235 %{
16236 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16237 match(Set result (EncodeISOArray src (Binary dst len)));
16238 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16239 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16240
16241 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16242 ins_encode %{
16243 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16244 $result$$Register, true,
16245 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16246 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16247 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16248 %}
16249 ins_pipe(pipe_class_memory);
16250 %}
16251
16252 //----------------------------- CompressBits/ExpandBits ------------------------
16253
16254 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16255 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16256 match(Set dst (CompressBits src mask));
16257 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16258 format %{ "mov $tsrc, $src\n\t"
16259 "mov $tmask, $mask\n\t"
16260 "bext $tdst, $tsrc, $tmask\n\t"
16261 "mov $dst, $tdst"
16262 %}
16263 ins_encode %{
16264 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16265 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16266 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16267 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16268 %}
16269 ins_pipe(pipe_slow);
16270 %}
16271
16272 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16273 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16274 match(Set dst (CompressBits (LoadI mem) mask));
16275 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16276 format %{ "ldrs $tsrc, $mem\n\t"
16277 "ldrs $tmask, $mask\n\t"
16278 "bext $tdst, $tsrc, $tmask\n\t"
16279 "mov $dst, $tdst"
16280 %}
16281 ins_encode %{
16282 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16283 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16284 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16285 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16286 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16287 %}
16288 ins_pipe(pipe_slow);
16289 %}
16290
16291 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16292 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16293 match(Set dst (CompressBits src mask));
16294 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16295 format %{ "mov $tsrc, $src\n\t"
16296 "mov $tmask, $mask\n\t"
16297 "bext $tdst, $tsrc, $tmask\n\t"
16298 "mov $dst, $tdst"
16299 %}
16300 ins_encode %{
16301 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16302 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16303 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16304 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16305 %}
16306 ins_pipe(pipe_slow);
16307 %}
16308
16309 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16310 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16311 match(Set dst (CompressBits (LoadL mem) mask));
16312 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16313 format %{ "ldrd $tsrc, $mem\n\t"
16314 "ldrd $tmask, $mask\n\t"
16315 "bext $tdst, $tsrc, $tmask\n\t"
16316 "mov $dst, $tdst"
16317 %}
16318 ins_encode %{
16319 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16320 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16321 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16322 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16323 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16324 %}
16325 ins_pipe(pipe_slow);
16326 %}
16327
16328 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16329 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16330 match(Set dst (ExpandBits src mask));
16331 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16332 format %{ "mov $tsrc, $src\n\t"
16333 "mov $tmask, $mask\n\t"
16334 "bdep $tdst, $tsrc, $tmask\n\t"
16335 "mov $dst, $tdst"
16336 %}
16337 ins_encode %{
16338 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16339 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16340 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16341 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16342 %}
16343 ins_pipe(pipe_slow);
16344 %}
16345
16346 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16347 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16348 match(Set dst (ExpandBits (LoadI mem) mask));
16349 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16350 format %{ "ldrs $tsrc, $mem\n\t"
16351 "ldrs $tmask, $mask\n\t"
16352 "bdep $tdst, $tsrc, $tmask\n\t"
16353 "mov $dst, $tdst"
16354 %}
16355 ins_encode %{
16356 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16357 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16358 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16359 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16360 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16361 %}
16362 ins_pipe(pipe_slow);
16363 %}
16364
16365 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16366 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16367 match(Set dst (ExpandBits src mask));
16368 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16369 format %{ "mov $tsrc, $src\n\t"
16370 "mov $tmask, $mask\n\t"
16371 "bdep $tdst, $tsrc, $tmask\n\t"
16372 "mov $dst, $tdst"
16373 %}
16374 ins_encode %{
16375 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16376 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16377 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16378 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16379 %}
16380 ins_pipe(pipe_slow);
16381 %}
16382
16383
16384 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16385 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16386 match(Set dst (ExpandBits (LoadL mem) mask));
16387 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16388 format %{ "ldrd $tsrc, $mem\n\t"
16389 "ldrd $tmask, $mask\n\t"
16390 "bdep $tdst, $tsrc, $tmask\n\t"
16391 "mov $dst, $tdst"
16392 %}
16393 ins_encode %{
16394 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16395 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16396 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16397 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16398 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16399 %}
16400 ins_pipe(pipe_slow);
16401 %}
16402
16403 //----------------------------- Reinterpret ----------------------------------
16404 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16405 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16406 match(Set dst (ReinterpretHF2S src));
16407 format %{ "reinterpretHF2S $dst, $src" %}
16408 ins_encode %{
16409 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16410 %}
16411 ins_pipe(pipe_slow);
16412 %}
16413
16414 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16415 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16416 match(Set dst (ReinterpretS2HF src));
16417 format %{ "reinterpretS2HF $dst, $src" %}
16418 ins_encode %{
16419 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16420 %}
16421 ins_pipe(pipe_slow);
16422 %}
16423
16424 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16425 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16426 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16427 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16428 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16429 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16430 // can be omitted in this pattern, resulting in -
16431 // fcvt $dst, $src // Convert float to half-precision float
16432 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16433 %{
16434 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16435 format %{ "convF2HFAndS2HF $dst, $src" %}
16436 ins_encode %{
16437 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16438 %}
16439 ins_pipe(pipe_slow);
16440 %}
16441
16442 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16443 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16444 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16445 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16446 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16447 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16448 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16449 // resulting in -
16450 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16451 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16452 %{
16453 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16454 format %{ "convHF2SAndHF2F $dst, $src" %}
16455 ins_encode %{
16456 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16457 %}
16458 ins_pipe(pipe_slow);
16459 %}
16460
16461 // ============================================================================
16462 // This name is KNOWN by the ADLC and cannot be changed.
16463 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16464 // for this guy.
16465 instruct tlsLoadP(thread_RegP dst)
16466 %{
16467 match(Set dst (ThreadLocal));
16468
16469 ins_cost(0);
16470
16471 format %{ " -- \t// $dst=Thread::current(), empty" %}
16472
16473 size(0);
16474
16475 ins_encode( /*empty*/ );
16476
16477 ins_pipe(pipe_class_empty);
16478 %}
16479
16480 //----------PEEPHOLE RULES-----------------------------------------------------
16481 // These must follow all instruction definitions as they use the names
16482 // defined in the instructions definitions.
16483 //
16484 // peepmatch ( root_instr_name [preceding_instruction]* );
16485 //
16486 // peepconstraint %{
16487 // (instruction_number.operand_name relational_op instruction_number.operand_name
16488 // [, ...] );
16489 // // instruction numbers are zero-based using left to right order in peepmatch
16490 //
16491 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16492 // // provide an instruction_number.operand_name for each operand that appears
16493 // // in the replacement instruction's match rule
16494 //
16495 // ---------VM FLAGS---------------------------------------------------------
16496 //
16497 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16498 //
16499 // Each peephole rule is given an identifying number starting with zero and
16500 // increasing by one in the order seen by the parser. An individual peephole
16501 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16502 // on the command-line.
16503 //
16504 // ---------CURRENT LIMITATIONS----------------------------------------------
16505 //
16506 // Only match adjacent instructions in same basic block
16507 // Only equality constraints
16508 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16509 // Only one replacement instruction
16510 //
16511 // ---------EXAMPLE----------------------------------------------------------
16512 //
16513 // // pertinent parts of existing instructions in architecture description
16514 // instruct movI(iRegINoSp dst, iRegI src)
16515 // %{
16516 // match(Set dst (CopyI src));
16517 // %}
16518 //
16519 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16520 // %{
16521 // match(Set dst (AddI dst src));
16522 // effect(KILL cr);
16523 // %}
16524 //
16525 // // Change (inc mov) to lea
16526 // peephole %{
16527 // // increment preceded by register-register move
16528 // peepmatch ( incI_iReg movI );
16529 // // require that the destination register of the increment
16530 // // match the destination register of the move
16531 // peepconstraint ( 0.dst == 1.dst );
16532 // // construct a replacement instruction that sets
16533 // // the destination to ( move's source register + one )
16534 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16535 // %}
16536 //
16537
16538 // Implementation no longer uses movX instructions since
16539 // machine-independent system no longer uses CopyX nodes.
16540 //
16541 // peephole
16542 // %{
16543 // peepmatch (incI_iReg movI);
16544 // peepconstraint (0.dst == 1.dst);
16545 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16546 // %}
16547
16548 // peephole
16549 // %{
16550 // peepmatch (decI_iReg movI);
16551 // peepconstraint (0.dst == 1.dst);
16552 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16553 // %}
16554
16555 // peephole
16556 // %{
16557 // peepmatch (addI_iReg_imm movI);
16558 // peepconstraint (0.dst == 1.dst);
16559 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16560 // %}
16561
16562 // peephole
16563 // %{
16564 // peepmatch (incL_iReg movL);
16565 // peepconstraint (0.dst == 1.dst);
16566 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16567 // %}
16568
16569 // peephole
16570 // %{
16571 // peepmatch (decL_iReg movL);
16572 // peepconstraint (0.dst == 1.dst);
16573 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16574 // %}
16575
16576 // peephole
16577 // %{
16578 // peepmatch (addL_iReg_imm movL);
16579 // peepconstraint (0.dst == 1.dst);
16580 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16581 // %}
16582
16583 // peephole
16584 // %{
16585 // peepmatch (addP_iReg_imm movP);
16586 // peepconstraint (0.dst == 1.dst);
16587 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16588 // %}
16589
16590 // // Change load of spilled value to only a spill
16591 // instruct storeI(memory mem, iRegI src)
16592 // %{
16593 // match(Set mem (StoreI mem src));
16594 // %}
16595 //
16596 // instruct loadI(iRegINoSp dst, memory mem)
16597 // %{
16598 // match(Set dst (LoadI mem));
16599 // %}
16600 //
16601
16602 //----------SMARTSPILL RULES---------------------------------------------------
16603 // These must follow all instruction definitions as they use the names
16604 // defined in the instructions definitions.
16605
16606 // Local Variables:
16607 // mode: c++
16608 // End: