1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2237 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2238 st->print_cr("\tcmpw rscratch1, r10");
2239 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2240 }
2241 #endif
2242
2243 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2244 {
2245 __ ic_check(InteriorEntryAlignment);
2246 }
2247
2248 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2249 {
2250 return MachNode::size(ra_);
2251 }
2252
2253 // REQUIRED EMIT CODE
2254
2255 //=============================================================================
2256
2257 // Emit deopt handler code.
2258 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2259 {
2260 // Note that the code buffer's insts_mark is always relative to insts.
2261 // That's why we must use the macroassembler to generate a handler.
2262 address base = __ start_a_stub(size_deopt_handler());
2263 if (base == nullptr) {
2264 ciEnv::current()->record_failure("CodeCache is full");
2265 return 0; // CodeBuffer::expand failed
2266 }
2267
2268 int offset = __ offset();
2269 Label start;
2270 __ bind(start);
2271 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2272
2273 int entry_offset = __ offset();
2274 __ b(start);
2275
2276 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2277 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2278 "out of bounds read in post-call NOP check");
2279 __ end_a_stub();
2280 return entry_offset;
2281 }
2282
2283 // REQUIRED MATCHER CODE
2284
2285 //=============================================================================
2286
2287 bool Matcher::match_rule_supported(int opcode) {
2288 if (!has_match_rule(opcode))
2289 return false;
2290
2291 switch (opcode) {
2292 case Op_OnSpinWait:
2293 return VM_Version::supports_on_spin_wait();
2294 case Op_CacheWB:
2295 case Op_CacheWBPreSync:
2296 case Op_CacheWBPostSync:
2297 if (!VM_Version::supports_data_cache_line_flush()) {
2298 return false;
2299 }
2300 break;
2301 case Op_ExpandBits:
2302 case Op_CompressBits:
2303 if (!VM_Version::supports_svebitperm()) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaF:
2308 case Op_FmaD:
2309 case Op_FmaVF:
2310 case Op_FmaVD:
2311 if (!UseFMA) {
2312 return false;
2313 }
2314 break;
2315 case Op_FmaHF:
2316 // UseFMA flag also needs to be checked along with FEAT_FP16
2317 if (!UseFMA || !is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 case Op_AddHF:
2322 case Op_SubHF:
2323 case Op_MulHF:
2324 case Op_DivHF:
2325 case Op_MinHF:
2326 case Op_MaxHF:
2327 case Op_SqrtHF:
2328 // Half-precision floating point scalar operations require FEAT_FP16
2329 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2330 // features are supported.
2331 if (!is_feat_fp16_supported()) {
2332 return false;
2333 }
2334 break;
2335 }
2336
2337 return true; // Per default match rules are supported.
2338 }
2339
2340 const RegMask* Matcher::predicate_reg_mask(void) {
2341 return &_PR_REG_mask;
2342 }
2343
2344 bool Matcher::supports_vector_calling_convention(void) {
2345 return EnableVectorSupport;
2346 }
2347
2348 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2349 assert(EnableVectorSupport, "sanity");
2350 int lo = V0_num;
2351 int hi = V0_H_num;
2352 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2353 hi = V0_K_num;
2354 }
2355 return OptoRegPair(hi, lo);
2356 }
2357
2358 // Is this branch offset short enough that a short branch can be used?
2359 //
2360 // NOTE: If the platform does not provide any short branch variants, then
2361 // this method should return false for offset 0.
2362 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2363 // The passed offset is relative to address of the branch.
2364
2365 return (-32768 <= offset && offset < 32768);
2366 }
2367
2368 // Vector width in bytes.
2369 int Matcher::vector_width_in_bytes(BasicType bt) {
2370 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2371 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2372 // Minimum 2 values in vector
2373 if (size < 2*type2aelembytes(bt)) size = 0;
2374 // But never < 4
2375 if (size < 4) size = 0;
2376 return size;
2377 }
2378
2379 // Limits on vector size (number of elements) loaded into vector.
2380 int Matcher::max_vector_size(const BasicType bt) {
2381 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2382 }
2383
2384 int Matcher::min_vector_size(const BasicType bt) {
2385 // Usually, the shortest vector length supported by AArch64 ISA and
2386 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2387 // vectors in a few special cases.
2388 int size;
2389 switch(bt) {
2390 case T_BOOLEAN:
2391 // Load/store a vector mask with only 2 elements for vector types
2392 // such as "2I/2F/2L/2D".
2393 size = 2;
2394 break;
2395 case T_BYTE:
2396 // Generate a "4B" vector, to support vector cast between "8B/16B"
2397 // and "4S/4I/4L/4F/4D".
2398 size = 4;
2399 break;
2400 case T_SHORT:
2401 // Generate a "2S" vector, to support vector cast between "4S/8S"
2402 // and "2I/2L/2F/2D".
2403 size = 2;
2404 break;
2405 default:
2406 // Limit the min vector length to 64-bit.
2407 size = 8 / type2aelembytes(bt);
2408 // The number of elements in a vector should be at least 2.
2409 size = MAX2(size, 2);
2410 }
2411
2412 int max_size = max_vector_size(bt);
2413 return MIN2(size, max_size);
2414 }
2415
2416 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2417 return Matcher::max_vector_size(bt);
2418 }
2419
2420 // Actual max scalable vector register length.
2421 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2422 return Matcher::max_vector_size(bt);
2423 }
2424
2425 // Vector ideal reg.
2426 uint Matcher::vector_ideal_reg(int len) {
2427 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2428 return Op_VecA;
2429 }
2430 switch(len) {
2431 // For 16-bit/32-bit mask vector, reuse VecD.
2432 case 2:
2433 case 4:
2434 case 8: return Op_VecD;
2435 case 16: return Op_VecX;
2436 }
2437 ShouldNotReachHere();
2438 return 0;
2439 }
2440
2441 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2442 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2443 switch (ideal_reg) {
2444 case Op_VecA: return new vecAOper();
2445 case Op_VecD: return new vecDOper();
2446 case Op_VecX: return new vecXOper();
2447 }
2448 ShouldNotReachHere();
2449 return nullptr;
2450 }
2451
2452 bool Matcher::is_reg2reg_move(MachNode* m) {
2453 return false;
2454 }
2455
2456 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2457 return false;
2458 }
2459
2460 bool Matcher::is_generic_vector(MachOper* opnd) {
2461 return opnd->opcode() == VREG;
2462 }
2463
2464 #ifdef ASSERT
2465 // Return whether or not this register is ever used as an argument.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486 #endif
2487
2488 uint Matcher::int_pressure_limit()
2489 {
2490 // JDK-8183543: When taking the number of available registers as int
2491 // register pressure threshold, the jtreg test:
2492 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2493 // failed due to C2 compilation failure with
2494 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2495 //
2496 // A derived pointer is live at CallNode and then is flagged by RA
2497 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2498 // derived pointers and lastly fail to spill after reaching maximum
2499 // number of iterations. Lowering the default pressure threshold to
2500 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2501 // a high register pressure area of the code so that split_DEF can
2502 // generate DefinitionSpillCopy for the derived pointer.
2503 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2504 if (!PreserveFramePointer) {
2505 // When PreserveFramePointer is off, frame pointer is allocatable,
2506 // but different from other SOC registers, it is excluded from
2507 // fatproj's mask because its save type is No-Save. Decrease 1 to
2508 // ensure high pressure at fatproj when PreserveFramePointer is off.
2509 // See check_pressure_at_fatproj().
2510 default_int_pressure_threshold--;
2511 }
2512 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2513 }
2514
2515 uint Matcher::float_pressure_limit()
2516 {
2517 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2518 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2519 }
2520
2521 const RegMask& Matcher::divI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for MODI projection of divmodI.
2527 const RegMask& Matcher::modI_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for DIVL projection of divmodL.
2533 const RegMask& Matcher::divL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 // Register for MODL projection of divmodL.
2539 const RegMask& Matcher::modL_proj_mask() {
2540 ShouldNotReachHere();
2541 return RegMask::EMPTY;
2542 }
2543
2544 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2545 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2546 Node* u = addp->fast_out(i);
2547 if (u->is_LoadStore()) {
2548 // On AArch64, LoadStoreNodes (i.e. compare and swap
2549 // instructions) only take register indirect as an operand, so
2550 // any attempt to use an AddPNode as an input to a LoadStoreNode
2551 // must fail.
2552 return false;
2553 }
2554 if (u->is_Mem()) {
2555 int opsize = u->as_Mem()->memory_size();
2556 assert(opsize > 0, "unexpected memory operand size");
2557 if (u->as_Mem()->memory_size() != (1<<shift)) {
2558 return false;
2559 }
2560 }
2561 }
2562 return true;
2563 }
2564
2565 // Convert BoolTest condition to Assembler condition.
2566 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2567 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2568 Assembler::Condition result;
2569 switch(cond) {
2570 case BoolTest::eq:
2571 result = Assembler::EQ; break;
2572 case BoolTest::ne:
2573 result = Assembler::NE; break;
2574 case BoolTest::le:
2575 result = Assembler::LE; break;
2576 case BoolTest::ge:
2577 result = Assembler::GE; break;
2578 case BoolTest::lt:
2579 result = Assembler::LT; break;
2580 case BoolTest::gt:
2581 result = Assembler::GT; break;
2582 case BoolTest::ule:
2583 result = Assembler::LS; break;
2584 case BoolTest::uge:
2585 result = Assembler::HS; break;
2586 case BoolTest::ult:
2587 result = Assembler::LO; break;
2588 case BoolTest::ugt:
2589 result = Assembler::HI; break;
2590 case BoolTest::overflow:
2591 result = Assembler::VS; break;
2592 case BoolTest::no_overflow:
2593 result = Assembler::VC; break;
2594 default:
2595 ShouldNotReachHere();
2596 return Assembler::Condition(-1);
2597 }
2598
2599 // Check conversion
2600 if (cond & BoolTest::unsigned_compare) {
2601 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2602 } else {
2603 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2604 }
2605
2606 return result;
2607 }
2608
2609 // Binary src (Replicate con)
2610 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2611 if (n == nullptr || m == nullptr) {
2612 return false;
2613 }
2614
2615 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2616 return false;
2617 }
2618
2619 Node* imm_node = m->in(1);
2620 if (!imm_node->is_Con()) {
2621 return false;
2622 }
2623
2624 const Type* t = imm_node->bottom_type();
2625 if (!(t->isa_int() || t->isa_long())) {
2626 return false;
2627 }
2628
2629 switch (n->Opcode()) {
2630 case Op_AndV:
2631 case Op_OrV:
2632 case Op_XorV: {
2633 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2634 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2635 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2636 }
2637 case Op_AddVB:
2638 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2639 case Op_AddVS:
2640 case Op_AddVI:
2641 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2642 case Op_AddVL:
2643 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2644 default:
2645 return false;
2646 }
2647 }
2648
2649 // (XorV src (Replicate m1))
2650 // (XorVMask src (MaskAll m1))
2651 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2652 if (n != nullptr && m != nullptr) {
2653 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2654 VectorNode::is_all_ones_vector(m);
2655 }
2656 return false;
2657 }
2658
2659 // Should the matcher clone input 'm' of node 'n'?
2660 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2661 if (is_vshift_con_pattern(n, m) ||
2662 is_vector_bitwise_not_pattern(n, m) ||
2663 is_valid_sve_arith_imm_pattern(n, m) ||
2664 is_encode_and_store_pattern(n, m)) {
2665 mstack.push(m, Visit);
2666 return true;
2667 }
2668 return false;
2669 }
2670
2671 // Should the Matcher clone shifts on addressing modes, expecting them
2672 // to be subsumed into complex addressing expressions or compute them
2673 // into registers?
2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2675
2676 // Loads and stores with indirect memory input (e.g., volatile loads and
2677 // stores) do not subsume the input into complex addressing expressions. If
2678 // the addressing expression is input to at least one such load or store, do
2679 // not clone the addressing expression. Query needs_acquiring_load and
2680 // needs_releasing_store as a proxy for indirect memory input, as it is not
2681 // possible to directly query for indirect memory input at this stage.
2682 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2683 Node* n = m->fast_out(i);
2684 if (n->is_Load() && needs_acquiring_load(n)) {
2685 return false;
2686 }
2687 if (n->is_Store() && needs_releasing_store(n)) {
2688 return false;
2689 }
2690 }
2691
2692 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2693 return true;
2694 }
2695
2696 Node *off = m->in(AddPNode::Offset);
2697 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2698 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2699 // Are there other uses besides address expressions?
2700 !is_visited(off)) {
2701 address_visited.set(off->_idx); // Flag as address_visited
2702 mstack.push(off->in(2), Visit);
2703 Node *conv = off->in(1);
2704 if (conv->Opcode() == Op_ConvI2L &&
2705 // Are there other uses besides address expressions?
2706 !is_visited(conv)) {
2707 address_visited.set(conv->_idx); // Flag as address_visited
2708 mstack.push(conv->in(1), Pre_Visit);
2709 } else {
2710 mstack.push(conv, Pre_Visit);
2711 }
2712 address_visited.test_set(m->_idx); // Flag as address_visited
2713 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2714 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2715 return true;
2716 } else if (off->Opcode() == Op_ConvI2L &&
2717 // Are there other uses besides address expressions?
2718 !is_visited(off)) {
2719 address_visited.test_set(m->_idx); // Flag as address_visited
2720 address_visited.set(off->_idx); // Flag as address_visited
2721 mstack.push(off->in(1), Pre_Visit);
2722 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2723 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2724 return true;
2725 }
2726 return false;
2727 }
2728
2729 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2730 { \
2731 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2732 guarantee(DISP == 0, "mode not permitted for volatile"); \
2733 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2734 __ INSN(REG, as_Register(BASE)); \
2735 }
2736
2737
2738 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2739 {
2740 Address::extend scale;
2741
2742 // Hooboy, this is fugly. We need a way to communicate to the
2743 // encoder that the index needs to be sign extended, so we have to
2744 // enumerate all the cases.
2745 switch (opcode) {
2746 case INDINDEXSCALEDI2L:
2747 case INDINDEXSCALEDI2LN:
2748 case INDINDEXI2L:
2749 case INDINDEXI2LN:
2750 scale = Address::sxtw(size);
2751 break;
2752 default:
2753 scale = Address::lsl(size);
2754 }
2755
2756 if (index == -1) {
2757 return Address(base, disp);
2758 } else {
2759 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2760 return Address(base, as_Register(index), scale);
2761 }
2762 }
2763
2764
2765 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2766 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2767 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2768 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2769 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2770
2771 // Used for all non-volatile memory accesses. The use of
2772 // $mem->opcode() to discover whether this pattern uses sign-extended
2773 // offsets is something of a kludge.
2774 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2775 Register reg, int opcode,
2776 Register base, int index, int scale, int disp,
2777 int size_in_memory)
2778 {
2779 Address addr = mem2address(opcode, base, index, scale, disp);
2780 if (addr.getMode() == Address::base_plus_offset) {
2781 /* Fix up any out-of-range offsets. */
2782 assert_different_registers(rscratch1, base);
2783 assert_different_registers(rscratch1, reg);
2784 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2785 }
2786 (masm->*insn)(reg, addr);
2787 }
2788
2789 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2790 FloatRegister reg, int opcode,
2791 Register base, int index, int size, int disp,
2792 int size_in_memory)
2793 {
2794 Address::extend scale;
2795
2796 switch (opcode) {
2797 case INDINDEXSCALEDI2L:
2798 case INDINDEXSCALEDI2LN:
2799 scale = Address::sxtw(size);
2800 break;
2801 default:
2802 scale = Address::lsl(size);
2803 }
2804
2805 if (index == -1) {
2806 // Fix up any out-of-range offsets.
2807 assert_different_registers(rscratch1, base);
2808 Address addr = Address(base, disp);
2809 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2810 (masm->*insn)(reg, addr);
2811 } else {
2812 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2813 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2814 }
2815 }
2816
2817 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2818 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2819 int opcode, Register base, int index, int size, int disp)
2820 {
2821 if (index == -1) {
2822 (masm->*insn)(reg, T, Address(base, disp));
2823 } else {
2824 assert(disp == 0, "unsupported address mode");
2825 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2826 }
2827 }
2828
2829 %}
2830
2831
2832
2833 //----------ENCODING BLOCK-----------------------------------------------------
2834 // This block specifies the encoding classes used by the compiler to
2835 // output byte streams. Encoding classes are parameterized macros
2836 // used by Machine Instruction Nodes in order to generate the bit
2837 // encoding of the instruction. Operands specify their base encoding
2838 // interface with the interface keyword. There are currently
2839 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2840 // COND_INTER. REG_INTER causes an operand to generate a function
2841 // which returns its register number when queried. CONST_INTER causes
2842 // an operand to generate a function which returns the value of the
2843 // constant when queried. MEMORY_INTER causes an operand to generate
2844 // four functions which return the Base Register, the Index Register,
2845 // the Scale Value, and the Offset Value of the operand when queried.
2846 // COND_INTER causes an operand to generate six functions which return
2847 // the encoding code (ie - encoding bits for the instruction)
2848 // associated with each basic boolean condition for a conditional
2849 // instruction.
2850 //
2851 // Instructions specify two basic values for encoding. Again, a
2852 // function is available to check if the constant displacement is an
2853 // oop. They use the ins_encode keyword to specify their encoding
2854 // classes (which must be a sequence of enc_class names, and their
2855 // parameters, specified in the encoding block), and they use the
2856 // opcode keyword to specify, in order, their primary, secondary, and
2857 // tertiary opcode. Only the opcode sections which a particular
2858 // instruction needs for encoding need to be specified.
2859 encode %{
2860 // Build emit functions for each basic byte or larger field in the
2861 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2862 // from C++ code in the enc_class source block. Emit functions will
2863 // live in the main source block for now. In future, we can
2864 // generalize this by adding a syntax that specifies the sizes of
2865 // fields in an order, so that the adlc can build the emit functions
2866 // automagically
2867
2868 // catch all for unimplemented encodings
2869 enc_class enc_unimplemented %{
2870 __ unimplemented("C2 catch all");
2871 %}
2872
2873 // BEGIN Non-volatile memory access
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2974 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2982 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2990 Register src_reg = as_Register($src$$reg);
2991 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strb0(memory1 mem) %{
2998 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3000 %}
3001
3002 // This encoding class is generated automatically from ad_encode.m4.
3003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3004 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3005 Register src_reg = as_Register($src$$reg);
3006 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_strh0(memory2 mem) %{
3013 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3014 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3015 %}
3016
3017 // This encoding class is generated automatically from ad_encode.m4.
3018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3019 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3020 Register src_reg = as_Register($src$$reg);
3021 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strw0(memory4 mem) %{
3028 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3029 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3030 %}
3031
3032 // This encoding class is generated automatically from ad_encode.m4.
3033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3034 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3035 Register src_reg = as_Register($src$$reg);
3036 // we sometimes get asked to store the stack pointer into the
3037 // current thread -- we cannot do that directly on AArch64
3038 if (src_reg == r31_sp) {
3039 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3040 __ mov(rscratch2, sp);
3041 src_reg = rscratch2;
3042 }
3043 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3044 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3045 %}
3046
3047 // This encoding class is generated automatically from ad_encode.m4.
3048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3049 enc_class aarch64_enc_str0(memory8 mem) %{
3050 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3051 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3052 %}
3053
3054 // This encoding class is generated automatically from ad_encode.m4.
3055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3056 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3057 FloatRegister src_reg = as_FloatRegister($src$$reg);
3058 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3065 FloatRegister src_reg = as_FloatRegister($src$$reg);
3066 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3073 __ membar(Assembler::StoreStore);
3074 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3076 %}
3077
3078 // END Non-volatile memory access
3079
3080 // Vector loads and stores
3081 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3082 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3083 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3084 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3085 %}
3086
3087 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3088 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3089 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3090 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3091 %}
3092
3093 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3100 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3101 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3106 FloatRegister src_reg = as_FloatRegister($src$$reg);
3107 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3112 FloatRegister src_reg = as_FloatRegister($src$$reg);
3113 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3124 FloatRegister src_reg = as_FloatRegister($src$$reg);
3125 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 // volatile loads and stores
3130
3131 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3132 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3133 rscratch1, stlrb);
3134 %}
3135
3136 enc_class aarch64_enc_stlrb0(memory mem) %{
3137 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, stlrb);
3139 %}
3140
3141 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3142 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, stlrh);
3144 %}
3145
3146 enc_class aarch64_enc_stlrh0(memory mem) %{
3147 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, stlrh);
3149 %}
3150
3151 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrw);
3154 %}
3155
3156 enc_class aarch64_enc_stlrw0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrw);
3159 %}
3160
3161 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3162 Register dst_reg = as_Register($dst$$reg);
3163 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldarb);
3165 __ sxtbw(dst_reg, dst_reg);
3166 %}
3167
3168 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3169 Register dst_reg = as_Register($dst$$reg);
3170 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarb);
3172 __ sxtb(dst_reg, dst_reg);
3173 %}
3174
3175 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3176 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, ldarb);
3178 %}
3179
3180 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3181 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarb);
3183 %}
3184
3185 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3186 Register dst_reg = as_Register($dst$$reg);
3187 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarh);
3189 __ sxthw(dst_reg, dst_reg);
3190 %}
3191
3192 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3193 Register dst_reg = as_Register($dst$$reg);
3194 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarh);
3196 __ sxth(dst_reg, dst_reg);
3197 %}
3198
3199 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3200 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3201 rscratch1, ldarh);
3202 %}
3203
3204 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3205 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, ldarh);
3207 %}
3208
3209 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3210 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarw);
3212 %}
3213
3214 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3215 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarw);
3217 %}
3218
3219 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldar);
3222 %}
3223
3224 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3225 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarw);
3227 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3228 %}
3229
3230 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3231 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldar);
3233 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3234 %}
3235
3236 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3237 Register src_reg = as_Register($src$$reg);
3238 // we sometimes get asked to store the stack pointer into the
3239 // current thread -- we cannot do that directly on AArch64
3240 if (src_reg == r31_sp) {
3241 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3242 __ mov(rscratch2, sp);
3243 src_reg = rscratch2;
3244 }
3245 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3246 rscratch1, stlr);
3247 %}
3248
3249 enc_class aarch64_enc_stlr0(memory mem) %{
3250 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3251 rscratch1, stlr);
3252 %}
3253
3254 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3255 {
3256 FloatRegister src_reg = as_FloatRegister($src$$reg);
3257 __ fmovs(rscratch2, src_reg);
3258 }
3259 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3260 rscratch1, stlrw);
3261 %}
3262
3263 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3264 {
3265 FloatRegister src_reg = as_FloatRegister($src$$reg);
3266 __ fmovd(rscratch2, src_reg);
3267 }
3268 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3269 rscratch1, stlr);
3270 %}
3271
3272 // synchronized read/update encodings
3273
3274 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3275 Register dst_reg = as_Register($dst$$reg);
3276 Register base = as_Register($mem$$base);
3277 int index = $mem$$index;
3278 int scale = $mem$$scale;
3279 int disp = $mem$$disp;
3280 if (index == -1) {
3281 if (disp != 0) {
3282 __ lea(rscratch1, Address(base, disp));
3283 __ ldaxr(dst_reg, rscratch1);
3284 } else {
3285 // TODO
3286 // should we ever get anything other than this case?
3287 __ ldaxr(dst_reg, base);
3288 }
3289 } else {
3290 Register index_reg = as_Register(index);
3291 if (disp == 0) {
3292 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3293 __ ldaxr(dst_reg, rscratch1);
3294 } else {
3295 __ lea(rscratch1, Address(base, disp));
3296 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3297 __ ldaxr(dst_reg, rscratch1);
3298 }
3299 }
3300 %}
3301
3302 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3303 Register src_reg = as_Register($src$$reg);
3304 Register base = as_Register($mem$$base);
3305 int index = $mem$$index;
3306 int scale = $mem$$scale;
3307 int disp = $mem$$disp;
3308 if (index == -1) {
3309 if (disp != 0) {
3310 __ lea(rscratch2, Address(base, disp));
3311 __ stlxr(rscratch1, src_reg, rscratch2);
3312 } else {
3313 // TODO
3314 // should we ever get anything other than this case?
3315 __ stlxr(rscratch1, src_reg, base);
3316 }
3317 } else {
3318 Register index_reg = as_Register(index);
3319 if (disp == 0) {
3320 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3321 __ stlxr(rscratch1, src_reg, rscratch2);
3322 } else {
3323 __ lea(rscratch2, Address(base, disp));
3324 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3325 __ stlxr(rscratch1, src_reg, rscratch2);
3326 }
3327 }
3328 __ cmpw(rscratch1, zr);
3329 %}
3330
3331 // prefetch encodings
3332
3333 enc_class aarch64_enc_prefetchw(memory mem) %{
3334 Register base = as_Register($mem$$base);
3335 int index = $mem$$index;
3336 int scale = $mem$$scale;
3337 int disp = $mem$$disp;
3338 if (index == -1) {
3339 // Fix up any out-of-range offsets.
3340 assert_different_registers(rscratch1, base);
3341 Address addr = Address(base, disp);
3342 addr = __ legitimize_address(addr, 8, rscratch1);
3343 __ prfm(addr, PSTL1KEEP);
3344 } else {
3345 Register index_reg = as_Register(index);
3346 if (disp == 0) {
3347 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3348 } else {
3349 __ lea(rscratch1, Address(base, disp));
3350 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3351 }
3352 }
3353 %}
3354
3355 // mov encodings
3356
3357 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3358 uint32_t con = (uint32_t)$src$$constant;
3359 Register dst_reg = as_Register($dst$$reg);
3360 if (con == 0) {
3361 __ movw(dst_reg, zr);
3362 } else {
3363 __ movw(dst_reg, con);
3364 }
3365 %}
3366
3367 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3368 Register dst_reg = as_Register($dst$$reg);
3369 uint64_t con = (uint64_t)$src$$constant;
3370 if (con == 0) {
3371 __ mov(dst_reg, zr);
3372 } else {
3373 __ mov(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 address con = (address)$src$$constant;
3380 if (con == nullptr || con == (address)1) {
3381 ShouldNotReachHere();
3382 } else {
3383 relocInfo::relocType rtype = $src->constant_reloc();
3384 if (rtype == relocInfo::oop_type) {
3385 __ movoop(dst_reg, (jobject)con);
3386 } else if (rtype == relocInfo::metadata_type) {
3387 __ mov_metadata(dst_reg, (Metadata*)con);
3388 } else {
3389 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3390 // load fake address constants using a normal move
3391 if (! __ is_valid_AArch64_address(con) ||
3392 con < (address)(uintptr_t)os::vm_page_size()) {
3393 __ mov(dst_reg, con);
3394 } else {
3395 // no reloc so just use adrp and add
3396 uint64_t offset;
3397 __ adrp(dst_reg, con, offset);
3398 __ add(dst_reg, dst_reg, offset);
3399 }
3400 }
3401 }
3402 %}
3403
3404 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3405 Register dst_reg = as_Register($dst$$reg);
3406 __ mov(dst_reg, zr);
3407 %}
3408
3409 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3410 Register dst_reg = as_Register($dst$$reg);
3411 __ mov(dst_reg, (uint64_t)1);
3412 %}
3413
3414 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3415 Register dst_reg = as_Register($dst$$reg);
3416 address con = (address)$src$$constant;
3417 if (con == nullptr) {
3418 ShouldNotReachHere();
3419 } else {
3420 relocInfo::relocType rtype = $src->constant_reloc();
3421 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3422 __ set_narrow_oop(dst_reg, (jobject)con);
3423 }
3424 %}
3425
3426 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, zr);
3429 %}
3430
3431 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3432 Register dst_reg = as_Register($dst$$reg);
3433 address con = (address)$src$$constant;
3434 if (con == nullptr) {
3435 ShouldNotReachHere();
3436 } else {
3437 relocInfo::relocType rtype = $src->constant_reloc();
3438 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3439 __ set_narrow_klass(dst_reg, (Klass *)con);
3440 }
3441 %}
3442
3443 // arithmetic encodings
3444
3445 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3446 Register dst_reg = as_Register($dst$$reg);
3447 Register src_reg = as_Register($src1$$reg);
3448 int32_t con = (int32_t)$src2$$constant;
3449 // add has primary == 0, subtract has primary == 1
3450 if ($primary) { con = -con; }
3451 if (con < 0) {
3452 __ subw(dst_reg, src_reg, -con);
3453 } else {
3454 __ addw(dst_reg, src_reg, con);
3455 }
3456 %}
3457
3458 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3459 Register dst_reg = as_Register($dst$$reg);
3460 Register src_reg = as_Register($src1$$reg);
3461 int32_t con = (int32_t)$src2$$constant;
3462 // add has primary == 0, subtract has primary == 1
3463 if ($primary) { con = -con; }
3464 if (con < 0) {
3465 __ sub(dst_reg, src_reg, -con);
3466 } else {
3467 __ add(dst_reg, src_reg, con);
3468 }
3469 %}
3470
3471 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3472 Register dst_reg = as_Register($dst$$reg);
3473 Register src1_reg = as_Register($src1$$reg);
3474 Register src2_reg = as_Register($src2$$reg);
3475 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3476 %}
3477
3478 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3479 Register dst_reg = as_Register($dst$$reg);
3480 Register src1_reg = as_Register($src1$$reg);
3481 Register src2_reg = as_Register($src2$$reg);
3482 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3483 %}
3484
3485 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3486 Register dst_reg = as_Register($dst$$reg);
3487 Register src1_reg = as_Register($src1$$reg);
3488 Register src2_reg = as_Register($src2$$reg);
3489 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3490 %}
3491
3492 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3497 %}
3498
3499 // compare instruction encodings
3500
3501 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3502 Register reg1 = as_Register($src1$$reg);
3503 Register reg2 = as_Register($src2$$reg);
3504 __ cmpw(reg1, reg2);
3505 %}
3506
3507 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3508 Register reg = as_Register($src1$$reg);
3509 int32_t val = $src2$$constant;
3510 if (val >= 0) {
3511 __ subsw(zr, reg, val);
3512 } else {
3513 __ addsw(zr, reg, -val);
3514 }
3515 %}
3516
3517 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3518 Register reg1 = as_Register($src1$$reg);
3519 uint32_t val = (uint32_t)$src2$$constant;
3520 __ movw(rscratch1, val);
3521 __ cmpw(reg1, rscratch1);
3522 %}
3523
3524 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3525 Register reg1 = as_Register($src1$$reg);
3526 Register reg2 = as_Register($src2$$reg);
3527 __ cmp(reg1, reg2);
3528 %}
3529
3530 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3531 Register reg = as_Register($src1$$reg);
3532 int64_t val = $src2$$constant;
3533 if (val >= 0) {
3534 __ subs(zr, reg, val);
3535 } else if (val != -val) {
3536 __ adds(zr, reg, -val);
3537 } else {
3538 // aargh, Long.MIN_VALUE is a special case
3539 __ orr(rscratch1, zr, (uint64_t)val);
3540 __ subs(zr, reg, rscratch1);
3541 }
3542 %}
3543
3544 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3545 Register reg1 = as_Register($src1$$reg);
3546 uint64_t val = (uint64_t)$src2$$constant;
3547 __ mov(rscratch1, val);
3548 __ cmp(reg1, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3552 Register reg1 = as_Register($src1$$reg);
3553 Register reg2 = as_Register($src2$$reg);
3554 __ cmp(reg1, reg2);
3555 %}
3556
3557 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 Register reg2 = as_Register($src2$$reg);
3560 __ cmpw(reg1, reg2);
3561 %}
3562
3563 enc_class aarch64_enc_testp(iRegP src) %{
3564 Register reg = as_Register($src$$reg);
3565 __ cmp(reg, zr);
3566 %}
3567
3568 enc_class aarch64_enc_testn(iRegN src) %{
3569 Register reg = as_Register($src$$reg);
3570 __ cmpw(reg, zr);
3571 %}
3572
3573 enc_class aarch64_enc_b(label lbl) %{
3574 Label *L = $lbl$$label;
3575 __ b(*L);
3576 %}
3577
3578 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3579 Label *L = $lbl$$label;
3580 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3581 %}
3582
3583 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3584 Label *L = $lbl$$label;
3585 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3586 %}
3587
3588 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3589 %{
3590 Register sub_reg = as_Register($sub$$reg);
3591 Register super_reg = as_Register($super$$reg);
3592 Register temp_reg = as_Register($temp$$reg);
3593 Register result_reg = as_Register($result$$reg);
3594
3595 Label miss;
3596 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3597 nullptr, &miss,
3598 /*set_cond_codes:*/ true);
3599 if ($primary) {
3600 __ mov(result_reg, zr);
3601 }
3602 __ bind(miss);
3603 %}
3604
3605 enc_class aarch64_enc_java_static_call(method meth) %{
3606 address addr = (address)$meth$$method;
3607 address call;
3608 if (!_method) {
3609 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3610 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3611 if (call == nullptr) {
3612 ciEnv::current()->record_failure("CodeCache is full");
3613 return;
3614 }
3615 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3616 // The NOP here is purely to ensure that eliding a call to
3617 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3618 __ nop();
3619 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3620 } else {
3621 int method_index = resolved_method_index(masm);
3622 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3623 : static_call_Relocation::spec(method_index);
3624 call = __ trampoline_call(Address(addr, rspec));
3625 if (call == nullptr) {
3626 ciEnv::current()->record_failure("CodeCache is full");
3627 return;
3628 }
3629 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3630 // Calls of the same statically bound method can share
3631 // a stub to the interpreter.
3632 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3633 } else {
3634 // Emit stub for static call
3635 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3636 if (stub == nullptr) {
3637 ciEnv::current()->record_failure("CodeCache is full");
3638 return;
3639 }
3640 }
3641 }
3642
3643 __ post_call_nop();
3644
3645 // Only non uncommon_trap calls need to reinitialize ptrue.
3646 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3647 __ reinitialize_ptrue();
3648 }
3649 %}
3650
3651 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3652 int method_index = resolved_method_index(masm);
3653 address call = __ ic_call((address)$meth$$method, method_index);
3654 if (call == nullptr) {
3655 ciEnv::current()->record_failure("CodeCache is full");
3656 return;
3657 }
3658 __ post_call_nop();
3659 if (Compile::current()->max_vector_size() > 0) {
3660 __ reinitialize_ptrue();
3661 }
3662 %}
3663
3664 enc_class aarch64_enc_call_epilog() %{
3665 if (VerifyStackAtCalls) {
3666 // Check that stack depth is unchanged: find majik cookie on stack
3667 __ call_Unimplemented();
3668 }
3669 %}
3670
3671 enc_class aarch64_enc_java_to_runtime(method meth) %{
3672 // some calls to generated routines (arraycopy code) are scheduled
3673 // by C2 as runtime calls. if so we can call them using a br (they
3674 // will be in a reachable segment) otherwise we have to use a blr
3675 // which loads the absolute address into a register.
3676 address entry = (address)$meth$$method;
3677 CodeBlob *cb = CodeCache::find_blob(entry);
3678 if (cb) {
3679 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3680 if (call == nullptr) {
3681 ciEnv::current()->record_failure("CodeCache is full");
3682 return;
3683 }
3684 __ post_call_nop();
3685 } else {
3686 Label retaddr;
3687 // Make the anchor frame walkable
3688 __ adr(rscratch2, retaddr);
3689 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3690 __ lea(rscratch1, RuntimeAddress(entry));
3691 __ blr(rscratch1);
3692 __ bind(retaddr);
3693 __ post_call_nop();
3694 }
3695 if (Compile::current()->max_vector_size() > 0) {
3696 __ reinitialize_ptrue();
3697 }
3698 %}
3699
3700 enc_class aarch64_enc_rethrow() %{
3701 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3702 %}
3703
3704 enc_class aarch64_enc_ret() %{
3705 #ifdef ASSERT
3706 if (Compile::current()->max_vector_size() > 0) {
3707 __ verify_ptrue();
3708 }
3709 #endif
3710 __ ret(lr);
3711 %}
3712
3713 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3714 Register target_reg = as_Register($jump_target$$reg);
3715 __ br(target_reg);
3716 %}
3717
3718 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3719 Register target_reg = as_Register($jump_target$$reg);
3720 // exception oop should be in r0
3721 // ret addr has been popped into lr
3722 // callee expects it in r3
3723 __ mov(r3, lr);
3724 __ br(target_reg);
3725 %}
3726
3727 %}
3728
3729 //----------FRAME--------------------------------------------------------------
3730 // Definition of frame structure and management information.
3731 //
3732 // S T A C K L A Y O U T Allocators stack-slot number
3733 // | (to get allocators register number
3734 // G Owned by | | v add OptoReg::stack0())
3735 // r CALLER | |
3736 // o | +--------+ pad to even-align allocators stack-slot
3737 // w V | pad0 | numbers; owned by CALLER
3738 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3739 // h ^ | in | 5
3740 // | | args | 4 Holes in incoming args owned by SELF
3741 // | | | | 3
3742 // | | +--------+
3743 // V | | old out| Empty on Intel, window on Sparc
3744 // | old |preserve| Must be even aligned.
3745 // | SP-+--------+----> Matcher::_old_SP, even aligned
3746 // | | in | 3 area for Intel ret address
3747 // Owned by |preserve| Empty on Sparc.
3748 // SELF +--------+
3749 // | | pad2 | 2 pad to align old SP
3750 // | +--------+ 1
3751 // | | locks | 0
3752 // | +--------+----> OptoReg::stack0(), even aligned
3753 // | | pad1 | 11 pad to align new SP
3754 // | +--------+
3755 // | | | 10
3756 // | | spills | 9 spills
3757 // V | | 8 (pad0 slot for callee)
3758 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3759 // ^ | out | 7
3760 // | | args | 6 Holes in outgoing args owned by CALLEE
3761 // Owned by +--------+
3762 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3763 // | new |preserve| Must be even-aligned.
3764 // | SP-+--------+----> Matcher::_new_SP, even aligned
3765 // | | |
3766 //
3767 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3768 // known from SELF's arguments and the Java calling convention.
3769 // Region 6-7 is determined per call site.
3770 // Note 2: If the calling convention leaves holes in the incoming argument
3771 // area, those holes are owned by SELF. Holes in the outgoing area
3772 // are owned by the CALLEE. Holes should not be necessary in the
3773 // incoming area, as the Java calling convention is completely under
3774 // the control of the AD file. Doubles can be sorted and packed to
3775 // avoid holes. Holes in the outgoing arguments may be necessary for
3776 // varargs C calling conventions.
3777 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3778 // even aligned with pad0 as needed.
3779 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3780 // (the latter is true on Intel but is it false on AArch64?)
3781 // region 6-11 is even aligned; it may be padded out more so that
3782 // the region from SP to FP meets the minimum stack alignment.
3783 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3784 // alignment. Region 11, pad1, may be dynamically extended so that
3785 // SP meets the minimum alignment.
3786
3787 frame %{
3788 // These three registers define part of the calling convention
3789 // between compiled code and the interpreter.
3790
3791 // Inline Cache Register or Method for I2C.
3792 inline_cache_reg(R12);
3793
3794 // Number of stack slots consumed by locking an object
3795 sync_stack_slots(2);
3796
3797 // Compiled code's Frame Pointer
3798 frame_pointer(R31);
3799
3800 // Stack alignment requirement
3801 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3802
3803 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3804 // for calls to C. Supports the var-args backing area for register parms.
3805 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3806
3807 // The after-PROLOG location of the return address. Location of
3808 // return address specifies a type (REG or STACK) and a number
3809 // representing the register number (i.e. - use a register name) or
3810 // stack slot.
3811 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3812 // Otherwise, it is above the locks and verification slot and alignment word
3813 // TODO this may well be correct but need to check why that - 2 is there
3814 // ppc port uses 0 but we definitely need to allow for fixed_slots
3815 // which folds in the space used for monitors
3816 return_addr(STACK - 2 +
3817 align_up((Compile::current()->in_preserve_stack_slots() +
3818 Compile::current()->fixed_slots()),
3819 stack_alignment_in_slots()));
3820
3821 // Location of compiled Java return values. Same as C for now.
3822 return_value
3823 %{
3824 // TODO do we allow ideal_reg == Op_RegN???
3825 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3826 "only return normal values");
3827
3828 static const int lo[Op_RegL + 1] = { // enum name
3829 0, // Op_Node
3830 0, // Op_Set
3831 R0_num, // Op_RegN
3832 R0_num, // Op_RegI
3833 R0_num, // Op_RegP
3834 V0_num, // Op_RegF
3835 V0_num, // Op_RegD
3836 R0_num // Op_RegL
3837 };
3838
3839 static const int hi[Op_RegL + 1] = { // enum name
3840 0, // Op_Node
3841 0, // Op_Set
3842 OptoReg::Bad, // Op_RegN
3843 OptoReg::Bad, // Op_RegI
3844 R0_H_num, // Op_RegP
3845 OptoReg::Bad, // Op_RegF
3846 V0_H_num, // Op_RegD
3847 R0_H_num // Op_RegL
3848 };
3849
3850 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3851 %}
3852 %}
3853
3854 //----------ATTRIBUTES---------------------------------------------------------
3855 //----------Operand Attributes-------------------------------------------------
3856 op_attrib op_cost(1); // Required cost attribute
3857
3858 //----------Instruction Attributes---------------------------------------------
3859 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3860 ins_attrib ins_size(32); // Required size attribute (in bits)
3861 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3862 // a non-matching short branch variant
3863 // of some long branch?
3864 ins_attrib ins_alignment(4); // Required alignment attribute (must
3865 // be a power of 2) specifies the
3866 // alignment that some part of the
3867 // instruction (not necessarily the
3868 // start) requires. If > 1, a
3869 // compute_padding() function must be
3870 // provided for the instruction
3871
3872 // Whether this node is expanded during code emission into a sequence of
3873 // instructions and the first instruction can perform an implicit null check.
3874 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3875
3876 //----------OPERANDS-----------------------------------------------------------
3877 // Operand definitions must precede instruction definitions for correct parsing
3878 // in the ADLC because operands constitute user defined types which are used in
3879 // instruction definitions.
3880
3881 //----------Simple Operands----------------------------------------------------
3882
3883 // Integer operands 32 bit
3884 // 32 bit immediate
3885 operand immI()
3886 %{
3887 match(ConI);
3888
3889 op_cost(0);
3890 format %{ %}
3891 interface(CONST_INTER);
3892 %}
3893
3894 // 32 bit zero
3895 operand immI0()
3896 %{
3897 predicate(n->get_int() == 0);
3898 match(ConI);
3899
3900 op_cost(0);
3901 format %{ %}
3902 interface(CONST_INTER);
3903 %}
3904
3905 // 32 bit unit increment
3906 operand immI_1()
3907 %{
3908 predicate(n->get_int() == 1);
3909 match(ConI);
3910
3911 op_cost(0);
3912 format %{ %}
3913 interface(CONST_INTER);
3914 %}
3915
3916 // 32 bit unit decrement
3917 operand immI_M1()
3918 %{
3919 predicate(n->get_int() == -1);
3920 match(ConI);
3921
3922 op_cost(0);
3923 format %{ %}
3924 interface(CONST_INTER);
3925 %}
3926
3927 // Shift values for add/sub extension shift
3928 operand immIExt()
3929 %{
3930 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 operand immI_gt_1()
3939 %{
3940 predicate(n->get_int() > 1);
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 operand immI_le_4()
3949 %{
3950 predicate(n->get_int() <= 4);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_16()
3959 %{
3960 predicate(n->get_int() == 16);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_24()
3969 %{
3970 predicate(n->get_int() == 24);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_32()
3979 %{
3980 predicate(n->get_int() == 32);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_48()
3989 %{
3990 predicate(n->get_int() == 48);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_56()
3999 %{
4000 predicate(n->get_int() == 56);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_255()
4009 %{
4010 predicate(n->get_int() == 255);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_65535()
4019 %{
4020 predicate(n->get_int() == 65535);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_positive()
4029 %{
4030 predicate(n->get_int() > 0);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 // BoolTest condition for signed compare
4039 operand immI_cmp_cond()
4040 %{
4041 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4042 match(ConI);
4043
4044 op_cost(0);
4045 format %{ %}
4046 interface(CONST_INTER);
4047 %}
4048
4049 // BoolTest condition for unsigned compare
4050 operand immI_cmpU_cond()
4051 %{
4052 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 operand immL_255()
4061 %{
4062 predicate(n->get_long() == 255L);
4063 match(ConL);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immL_65535()
4071 %{
4072 predicate(n->get_long() == 65535L);
4073 match(ConL);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immL_4294967295()
4081 %{
4082 predicate(n->get_long() == 4294967295L);
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_bitmask()
4091 %{
4092 predicate((n->get_long() != 0)
4093 && ((n->get_long() & 0xc000000000000000l) == 0)
4094 && is_power_of_2(n->get_long() + 1));
4095 match(ConL);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 operand immI_bitmask()
4103 %{
4104 predicate((n->get_int() != 0)
4105 && ((n->get_int() & 0xc0000000) == 0)
4106 && is_power_of_2(n->get_int() + 1));
4107 match(ConI);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immL_positive_bitmaskI()
4115 %{
4116 predicate((n->get_long() != 0)
4117 && ((julong)n->get_long() < 0x80000000ULL)
4118 && is_power_of_2(n->get_long() + 1));
4119 match(ConL);
4120
4121 op_cost(0);
4122 format %{ %}
4123 interface(CONST_INTER);
4124 %}
4125
4126 // Scale values for scaled offset addressing modes (up to long but not quad)
4127 operand immIScale()
4128 %{
4129 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4130 match(ConI);
4131
4132 op_cost(0);
4133 format %{ %}
4134 interface(CONST_INTER);
4135 %}
4136
4137 // 5 bit signed integer
4138 operand immI5()
4139 %{
4140 predicate(Assembler::is_simm(n->get_int(), 5));
4141 match(ConI);
4142
4143 op_cost(0);
4144 format %{ %}
4145 interface(CONST_INTER);
4146 %}
4147
4148 // 7 bit unsigned integer
4149 operand immIU7()
4150 %{
4151 predicate(Assembler::is_uimm(n->get_int(), 7));
4152 match(ConI);
4153
4154 op_cost(0);
4155 format %{ %}
4156 interface(CONST_INTER);
4157 %}
4158
4159 // Offset for scaled or unscaled immediate loads and stores
4160 operand immIOffset()
4161 %{
4162 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4163 match(ConI);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 operand immIOffset1()
4171 %{
4172 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immIOffset2()
4181 %{
4182 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immIOffset4()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset8()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset16()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immLOffset()
4221 %{
4222 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4223 match(ConL);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immLoffset1()
4231 %{
4232 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4233 match(ConL);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immLoffset2()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4243 match(ConL);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLoffset4()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset8()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset16()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 // 5 bit signed long integer
4281 operand immL5()
4282 %{
4283 predicate(Assembler::is_simm(n->get_long(), 5));
4284 match(ConL);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 // 7 bit unsigned long integer
4292 operand immLU7()
4293 %{
4294 predicate(Assembler::is_uimm(n->get_long(), 7));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 // 8 bit signed value.
4303 operand immI8()
4304 %{
4305 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4306 match(ConI);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 // 8 bit signed value (simm8), or #simm8 LSL 8.
4314 operand immIDupV()
4315 %{
4316 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4317 match(ConI);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 8 bit signed value (simm8), or #simm8 LSL 8.
4325 operand immLDupV()
4326 %{
4327 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4328 match(ConL);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 8 bit signed value (simm8), or #simm8 LSL 8.
4336 operand immHDupV()
4337 %{
4338 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4339 match(ConH);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit integer valid for vector add sub immediate
4347 operand immBAddSubV()
4348 %{
4349 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4350 match(ConI);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 // 32 bit integer valid for add sub immediate
4358 operand immIAddSub()
4359 %{
4360 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4361 match(ConI);
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 32 bit integer valid for vector add sub immediate
4368 operand immIAddSubV()
4369 %{
4370 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 32 bit unsigned integer valid for logical immediate
4379
4380 operand immBLog()
4381 %{
4382 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4383 match(ConI);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 operand immSLog()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4393 match(ConI);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 operand immILog()
4401 %{
4402 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 // Integer operands 64 bit
4411 // 64 bit immediate
4412 operand immL()
4413 %{
4414 match(ConL);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 64 bit zero
4422 operand immL0()
4423 %{
4424 predicate(n->get_long() == 0);
4425 match(ConL);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 // 64 bit unit decrement
4433 operand immL_M1()
4434 %{
4435 predicate(n->get_long() == -1);
4436 match(ConL);
4437
4438 op_cost(0);
4439 format %{ %}
4440 interface(CONST_INTER);
4441 %}
4442
4443 // 64 bit integer valid for add sub immediate
4444 operand immLAddSub()
4445 %{
4446 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4447 match(ConL);
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit integer valid for addv subv immediate
4454 operand immLAddSubV()
4455 %{
4456 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit integer valid for logical immediate
4465 operand immLLog()
4466 %{
4467 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4468 match(ConL);
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // Long Immediate: low 32-bit mask
4475 operand immL_32bits()
4476 %{
4477 predicate(n->get_long() == 0xFFFFFFFFL);
4478 match(ConL);
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // Pointer operands
4485 // Pointer Immediate
4486 operand immP()
4487 %{
4488 match(ConP);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // nullptr Pointer Immediate
4496 operand immP0()
4497 %{
4498 predicate(n->get_ptr() == 0);
4499 match(ConP);
4500
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 // Pointer Immediate One
4507 // this is used in object initialization (initial object header)
4508 operand immP_1()
4509 %{
4510 predicate(n->get_ptr() == 1);
4511 match(ConP);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // AOT Runtime Constants Address
4519 operand immAOTRuntimeConstantsAddress()
4520 %{
4521 // Check if the address is in the range of AOT Runtime Constants
4522 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4523 match(ConP);
4524
4525 op_cost(0);
4526 format %{ %}
4527 interface(CONST_INTER);
4528 %}
4529
4530 // Float and Double operands
4531 // Double Immediate
4532 operand immD()
4533 %{
4534 match(ConD);
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // Double Immediate: +0.0d
4541 operand immD0()
4542 %{
4543 predicate(jlong_cast(n->getd()) == 0);
4544 match(ConD);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 // constant 'double +0.0'.
4552 operand immDPacked()
4553 %{
4554 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4555 match(ConD);
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // Float Immediate
4562 operand immF()
4563 %{
4564 match(ConF);
4565 op_cost(0);
4566 format %{ %}
4567 interface(CONST_INTER);
4568 %}
4569
4570 // Float Immediate: +0.0f.
4571 operand immF0()
4572 %{
4573 predicate(jint_cast(n->getf()) == 0);
4574 match(ConF);
4575
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // Half Float (FP16) Immediate
4582 operand immH()
4583 %{
4584 match(ConH);
4585 op_cost(0);
4586 format %{ %}
4587 interface(CONST_INTER);
4588 %}
4589
4590 //
4591 operand immFPacked()
4592 %{
4593 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4594 match(ConF);
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 // Narrow pointer operands
4601 // Narrow Pointer Immediate
4602 operand immN()
4603 %{
4604 match(ConN);
4605
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // Narrow nullptr Pointer Immediate
4612 operand immN0()
4613 %{
4614 predicate(n->get_narrowcon() == 0);
4615 match(ConN);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 operand immNKlass()
4623 %{
4624 match(ConNKlass);
4625
4626 op_cost(0);
4627 format %{ %}
4628 interface(CONST_INTER);
4629 %}
4630
4631 // Integer 32 bit Register Operands
4632 // Integer 32 bitRegister (excludes SP)
4633 operand iRegI()
4634 %{
4635 constraint(ALLOC_IN_RC(any_reg32));
4636 match(RegI);
4637 match(iRegINoSp);
4638 op_cost(0);
4639 format %{ %}
4640 interface(REG_INTER);
4641 %}
4642
4643 // Integer 32 bit Register not Special
4644 operand iRegINoSp()
4645 %{
4646 constraint(ALLOC_IN_RC(no_special_reg32));
4647 match(RegI);
4648 op_cost(0);
4649 format %{ %}
4650 interface(REG_INTER);
4651 %}
4652
4653 // Integer 64 bit Register Operands
4654 // Integer 64 bit Register (includes SP)
4655 operand iRegL()
4656 %{
4657 constraint(ALLOC_IN_RC(any_reg));
4658 match(RegL);
4659 match(iRegLNoSp);
4660 op_cost(0);
4661 format %{ %}
4662 interface(REG_INTER);
4663 %}
4664
4665 // Integer 64 bit Register not Special
4666 operand iRegLNoSp()
4667 %{
4668 constraint(ALLOC_IN_RC(no_special_reg));
4669 match(RegL);
4670 match(iRegL_R0);
4671 format %{ %}
4672 interface(REG_INTER);
4673 %}
4674
4675 // Pointer Register Operands
4676 // Pointer Register
4677 operand iRegP()
4678 %{
4679 constraint(ALLOC_IN_RC(ptr_reg));
4680 match(RegP);
4681 match(iRegPNoSp);
4682 match(iRegP_R0);
4683 //match(iRegP_R2);
4684 //match(iRegP_R4);
4685 match(iRegP_R5);
4686 match(thread_RegP);
4687 op_cost(0);
4688 format %{ %}
4689 interface(REG_INTER);
4690 %}
4691
4692 // Pointer 64 bit Register not Special
4693 operand iRegPNoSp()
4694 %{
4695 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4696 match(RegP);
4697 // match(iRegP);
4698 // match(iRegP_R0);
4699 // match(iRegP_R2);
4700 // match(iRegP_R4);
4701 // match(iRegP_R5);
4702 // match(thread_RegP);
4703 op_cost(0);
4704 format %{ %}
4705 interface(REG_INTER);
4706 %}
4707
4708 // This operand is not allowed to use rfp even if
4709 // rfp is not used to hold the frame pointer.
4710 operand iRegPNoSpNoRfp()
4711 %{
4712 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4713 match(RegP);
4714 match(iRegPNoSp);
4715 op_cost(0);
4716 format %{ %}
4717 interface(REG_INTER);
4718 %}
4719
4720 // Pointer 64 bit Register R0 only
4721 operand iRegP_R0()
4722 %{
4723 constraint(ALLOC_IN_RC(r0_reg));
4724 match(RegP);
4725 // match(iRegP);
4726 match(iRegPNoSp);
4727 op_cost(0);
4728 format %{ %}
4729 interface(REG_INTER);
4730 %}
4731
4732 // Pointer 64 bit Register R1 only
4733 operand iRegP_R1()
4734 %{
4735 constraint(ALLOC_IN_RC(r1_reg));
4736 match(RegP);
4737 // match(iRegP);
4738 match(iRegPNoSp);
4739 op_cost(0);
4740 format %{ %}
4741 interface(REG_INTER);
4742 %}
4743
4744 // Pointer 64 bit Register R2 only
4745 operand iRegP_R2()
4746 %{
4747 constraint(ALLOC_IN_RC(r2_reg));
4748 match(RegP);
4749 // match(iRegP);
4750 match(iRegPNoSp);
4751 op_cost(0);
4752 format %{ %}
4753 interface(REG_INTER);
4754 %}
4755
4756 // Pointer 64 bit Register R3 only
4757 operand iRegP_R3()
4758 %{
4759 constraint(ALLOC_IN_RC(r3_reg));
4760 match(RegP);
4761 // match(iRegP);
4762 match(iRegPNoSp);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 // Pointer 64 bit Register R4 only
4769 operand iRegP_R4()
4770 %{
4771 constraint(ALLOC_IN_RC(r4_reg));
4772 match(RegP);
4773 // match(iRegP);
4774 match(iRegPNoSp);
4775 op_cost(0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer 64 bit Register R5 only
4781 operand iRegP_R5()
4782 %{
4783 constraint(ALLOC_IN_RC(r5_reg));
4784 match(RegP);
4785 // match(iRegP);
4786 match(iRegPNoSp);
4787 op_cost(0);
4788 format %{ %}
4789 interface(REG_INTER);
4790 %}
4791
4792 // Pointer 64 bit Register R10 only
4793 operand iRegP_R10()
4794 %{
4795 constraint(ALLOC_IN_RC(r10_reg));
4796 match(RegP);
4797 // match(iRegP);
4798 match(iRegPNoSp);
4799 op_cost(0);
4800 format %{ %}
4801 interface(REG_INTER);
4802 %}
4803
4804 // Long 64 bit Register R0 only
4805 operand iRegL_R0()
4806 %{
4807 constraint(ALLOC_IN_RC(r0_reg));
4808 match(RegL);
4809 match(iRegLNoSp);
4810 op_cost(0);
4811 format %{ %}
4812 interface(REG_INTER);
4813 %}
4814
4815 // Long 64 bit Register R11 only
4816 operand iRegL_R11()
4817 %{
4818 constraint(ALLOC_IN_RC(r11_reg));
4819 match(RegL);
4820 match(iRegLNoSp);
4821 op_cost(0);
4822 format %{ %}
4823 interface(REG_INTER);
4824 %}
4825
4826 // Register R0 only
4827 operand iRegI_R0()
4828 %{
4829 constraint(ALLOC_IN_RC(int_r0_reg));
4830 match(RegI);
4831 match(iRegINoSp);
4832 op_cost(0);
4833 format %{ %}
4834 interface(REG_INTER);
4835 %}
4836
4837 // Register R2 only
4838 operand iRegI_R2()
4839 %{
4840 constraint(ALLOC_IN_RC(int_r2_reg));
4841 match(RegI);
4842 match(iRegINoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Register R3 only
4849 operand iRegI_R3()
4850 %{
4851 constraint(ALLOC_IN_RC(int_r3_reg));
4852 match(RegI);
4853 match(iRegINoSp);
4854 op_cost(0);
4855 format %{ %}
4856 interface(REG_INTER);
4857 %}
4858
4859
4860 // Register R4 only
4861 operand iRegI_R4()
4862 %{
4863 constraint(ALLOC_IN_RC(int_r4_reg));
4864 match(RegI);
4865 match(iRegINoSp);
4866 op_cost(0);
4867 format %{ %}
4868 interface(REG_INTER);
4869 %}
4870
4871
4872 // Pointer Register Operands
4873 // Narrow Pointer Register
4874 operand iRegN()
4875 %{
4876 constraint(ALLOC_IN_RC(any_reg32));
4877 match(RegN);
4878 match(iRegNNoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884 // Integer 64 bit Register not Special
4885 operand iRegNNoSp()
4886 %{
4887 constraint(ALLOC_IN_RC(no_special_reg32));
4888 match(RegN);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Float Register
4895 // Float register operands
4896 operand vRegF()
4897 %{
4898 constraint(ALLOC_IN_RC(float_reg));
4899 match(RegF);
4900
4901 op_cost(0);
4902 format %{ %}
4903 interface(REG_INTER);
4904 %}
4905
4906 // Double Register
4907 // Double register operands
4908 operand vRegD()
4909 %{
4910 constraint(ALLOC_IN_RC(double_reg));
4911 match(RegD);
4912
4913 op_cost(0);
4914 format %{ %}
4915 interface(REG_INTER);
4916 %}
4917
4918 // Generic vector class. This will be used for
4919 // all vector operands, including NEON and SVE.
4920 operand vReg()
4921 %{
4922 constraint(ALLOC_IN_RC(dynamic));
4923 match(VecA);
4924 match(VecD);
4925 match(VecX);
4926
4927 op_cost(0);
4928 format %{ %}
4929 interface(REG_INTER);
4930 %}
4931
4932 operand vReg_V10()
4933 %{
4934 constraint(ALLOC_IN_RC(v10_veca_reg));
4935 match(vReg);
4936
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 operand vReg_V11()
4943 %{
4944 constraint(ALLOC_IN_RC(v11_veca_reg));
4945 match(vReg);
4946
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 operand vReg_V12()
4953 %{
4954 constraint(ALLOC_IN_RC(v12_veca_reg));
4955 match(vReg);
4956
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vReg_V13()
4963 %{
4964 constraint(ALLOC_IN_RC(v13_veca_reg));
4965 match(vReg);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vReg_V17()
4973 %{
4974 constraint(ALLOC_IN_RC(v17_veca_reg));
4975 match(vReg);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vReg_V18()
4983 %{
4984 constraint(ALLOC_IN_RC(v18_veca_reg));
4985 match(vReg);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V23()
4993 %{
4994 constraint(ALLOC_IN_RC(v23_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V24()
5003 %{
5004 constraint(ALLOC_IN_RC(v24_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vecA()
5013 %{
5014 constraint(ALLOC_IN_RC(vectora_reg));
5015 match(VecA);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vecD()
5023 %{
5024 constraint(ALLOC_IN_RC(vectord_reg));
5025 match(VecD);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vecX()
5033 %{
5034 constraint(ALLOC_IN_RC(vectorx_reg));
5035 match(VecX);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vRegD_V0()
5043 %{
5044 constraint(ALLOC_IN_RC(v0_reg));
5045 match(RegD);
5046 op_cost(0);
5047 format %{ %}
5048 interface(REG_INTER);
5049 %}
5050
5051 operand vRegD_V1()
5052 %{
5053 constraint(ALLOC_IN_RC(v1_reg));
5054 match(RegD);
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060 operand vRegD_V2()
5061 %{
5062 constraint(ALLOC_IN_RC(v2_reg));
5063 match(RegD);
5064 op_cost(0);
5065 format %{ %}
5066 interface(REG_INTER);
5067 %}
5068
5069 operand vRegD_V3()
5070 %{
5071 constraint(ALLOC_IN_RC(v3_reg));
5072 match(RegD);
5073 op_cost(0);
5074 format %{ %}
5075 interface(REG_INTER);
5076 %}
5077
5078 operand vRegD_V4()
5079 %{
5080 constraint(ALLOC_IN_RC(v4_reg));
5081 match(RegD);
5082 op_cost(0);
5083 format %{ %}
5084 interface(REG_INTER);
5085 %}
5086
5087 operand vRegD_V5()
5088 %{
5089 constraint(ALLOC_IN_RC(v5_reg));
5090 match(RegD);
5091 op_cost(0);
5092 format %{ %}
5093 interface(REG_INTER);
5094 %}
5095
5096 operand vRegD_V6()
5097 %{
5098 constraint(ALLOC_IN_RC(v6_reg));
5099 match(RegD);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105 operand vRegD_V7()
5106 %{
5107 constraint(ALLOC_IN_RC(v7_reg));
5108 match(RegD);
5109 op_cost(0);
5110 format %{ %}
5111 interface(REG_INTER);
5112 %}
5113
5114 operand vRegD_V12()
5115 %{
5116 constraint(ALLOC_IN_RC(v12_reg));
5117 match(RegD);
5118 op_cost(0);
5119 format %{ %}
5120 interface(REG_INTER);
5121 %}
5122
5123 operand vRegD_V13()
5124 %{
5125 constraint(ALLOC_IN_RC(v13_reg));
5126 match(RegD);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 operand pReg()
5133 %{
5134 constraint(ALLOC_IN_RC(pr_reg));
5135 match(RegVectMask);
5136 match(pRegGov);
5137 op_cost(0);
5138 format %{ %}
5139 interface(REG_INTER);
5140 %}
5141
5142 operand pRegGov()
5143 %{
5144 constraint(ALLOC_IN_RC(gov_pr));
5145 match(RegVectMask);
5146 match(pReg);
5147 op_cost(0);
5148 format %{ %}
5149 interface(REG_INTER);
5150 %}
5151
5152 operand pRegGov_P0()
5153 %{
5154 constraint(ALLOC_IN_RC(p0_reg));
5155 match(RegVectMask);
5156 op_cost(0);
5157 format %{ %}
5158 interface(REG_INTER);
5159 %}
5160
5161 operand pRegGov_P1()
5162 %{
5163 constraint(ALLOC_IN_RC(p1_reg));
5164 match(RegVectMask);
5165 op_cost(0);
5166 format %{ %}
5167 interface(REG_INTER);
5168 %}
5169
5170 // Flags register, used as output of signed compare instructions
5171
5172 // note that on AArch64 we also use this register as the output for
5173 // for floating point compare instructions (CmpF CmpD). this ensures
5174 // that ordered inequality tests use GT, GE, LT or LE none of which
5175 // pass through cases where the result is unordered i.e. one or both
5176 // inputs to the compare is a NaN. this means that the ideal code can
5177 // replace e.g. a GT with an LE and not end up capturing the NaN case
5178 // (where the comparison should always fail). EQ and NE tests are
5179 // always generated in ideal code so that unordered folds into the NE
5180 // case, matching the behaviour of AArch64 NE.
5181 //
5182 // This differs from x86 where the outputs of FP compares use a
5183 // special FP flags registers and where compares based on this
5184 // register are distinguished into ordered inequalities (cmpOpUCF) and
5185 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5186 // to explicitly handle the unordered case in branches. x86 also has
5187 // to include extra CMoveX rules to accept a cmpOpUCF input.
5188
5189 operand rFlagsReg()
5190 %{
5191 constraint(ALLOC_IN_RC(int_flags));
5192 match(RegFlags);
5193
5194 op_cost(0);
5195 format %{ "RFLAGS" %}
5196 interface(REG_INTER);
5197 %}
5198
5199 // Flags register, used as output of unsigned compare instructions
5200 operand rFlagsRegU()
5201 %{
5202 constraint(ALLOC_IN_RC(int_flags));
5203 match(RegFlags);
5204
5205 op_cost(0);
5206 format %{ "RFLAGSU" %}
5207 interface(REG_INTER);
5208 %}
5209
5210 // Special Registers
5211
5212 // Method Register
5213 operand inline_cache_RegP(iRegP reg)
5214 %{
5215 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5216 match(reg);
5217 match(iRegPNoSp);
5218 op_cost(0);
5219 format %{ %}
5220 interface(REG_INTER);
5221 %}
5222
5223 // Thread Register
5224 operand thread_RegP(iRegP reg)
5225 %{
5226 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5227 match(reg);
5228 op_cost(0);
5229 format %{ %}
5230 interface(REG_INTER);
5231 %}
5232
5233 //----------Memory Operands----------------------------------------------------
5234
5235 operand indirect(iRegP reg)
5236 %{
5237 constraint(ALLOC_IN_RC(ptr_reg));
5238 match(reg);
5239 op_cost(0);
5240 format %{ "[$reg]" %}
5241 interface(MEMORY_INTER) %{
5242 base($reg);
5243 index(0xffffffff);
5244 scale(0x0);
5245 disp(0x0);
5246 %}
5247 %}
5248
5249 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5250 %{
5251 constraint(ALLOC_IN_RC(ptr_reg));
5252 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5253 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5254 op_cost(0);
5255 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5256 interface(MEMORY_INTER) %{
5257 base($reg);
5258 index($ireg);
5259 scale($scale);
5260 disp(0x0);
5261 %}
5262 %}
5263
5264 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5265 %{
5266 constraint(ALLOC_IN_RC(ptr_reg));
5267 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5268 match(AddP reg (LShiftL lreg scale));
5269 op_cost(0);
5270 format %{ "$reg, $lreg lsl($scale)" %}
5271 interface(MEMORY_INTER) %{
5272 base($reg);
5273 index($lreg);
5274 scale($scale);
5275 disp(0x0);
5276 %}
5277 %}
5278
5279 operand indIndexI2L(iRegP reg, iRegI ireg)
5280 %{
5281 constraint(ALLOC_IN_RC(ptr_reg));
5282 match(AddP reg (ConvI2L ireg));
5283 op_cost(0);
5284 format %{ "$reg, $ireg, 0, I2L" %}
5285 interface(MEMORY_INTER) %{
5286 base($reg);
5287 index($ireg);
5288 scale(0x0);
5289 disp(0x0);
5290 %}
5291 %}
5292
5293 operand indIndex(iRegP reg, iRegL lreg)
5294 %{
5295 constraint(ALLOC_IN_RC(ptr_reg));
5296 match(AddP reg lreg);
5297 op_cost(0);
5298 format %{ "$reg, $lreg" %}
5299 interface(MEMORY_INTER) %{
5300 base($reg);
5301 index($lreg);
5302 scale(0x0);
5303 disp(0x0);
5304 %}
5305 %}
5306
5307 operand indOffI1(iRegP reg, immIOffset1 off)
5308 %{
5309 constraint(ALLOC_IN_RC(ptr_reg));
5310 match(AddP reg off);
5311 op_cost(0);
5312 format %{ "[$reg, $off]" %}
5313 interface(MEMORY_INTER) %{
5314 base($reg);
5315 index(0xffffffff);
5316 scale(0x0);
5317 disp($off);
5318 %}
5319 %}
5320
5321 operand indOffI2(iRegP reg, immIOffset2 off)
5322 %{
5323 constraint(ALLOC_IN_RC(ptr_reg));
5324 match(AddP reg off);
5325 op_cost(0);
5326 format %{ "[$reg, $off]" %}
5327 interface(MEMORY_INTER) %{
5328 base($reg);
5329 index(0xffffffff);
5330 scale(0x0);
5331 disp($off);
5332 %}
5333 %}
5334
5335 operand indOffI4(iRegP reg, immIOffset4 off)
5336 %{
5337 constraint(ALLOC_IN_RC(ptr_reg));
5338 match(AddP reg off);
5339 op_cost(0);
5340 format %{ "[$reg, $off]" %}
5341 interface(MEMORY_INTER) %{
5342 base($reg);
5343 index(0xffffffff);
5344 scale(0x0);
5345 disp($off);
5346 %}
5347 %}
5348
5349 operand indOffI8(iRegP reg, immIOffset8 off)
5350 %{
5351 constraint(ALLOC_IN_RC(ptr_reg));
5352 match(AddP reg off);
5353 op_cost(0);
5354 format %{ "[$reg, $off]" %}
5355 interface(MEMORY_INTER) %{
5356 base($reg);
5357 index(0xffffffff);
5358 scale(0x0);
5359 disp($off);
5360 %}
5361 %}
5362
5363 operand indOffI16(iRegP reg, immIOffset16 off)
5364 %{
5365 constraint(ALLOC_IN_RC(ptr_reg));
5366 match(AddP reg off);
5367 op_cost(0);
5368 format %{ "[$reg, $off]" %}
5369 interface(MEMORY_INTER) %{
5370 base($reg);
5371 index(0xffffffff);
5372 scale(0x0);
5373 disp($off);
5374 %}
5375 %}
5376
5377 operand indOffL1(iRegP reg, immLoffset1 off)
5378 %{
5379 constraint(ALLOC_IN_RC(ptr_reg));
5380 match(AddP reg off);
5381 op_cost(0);
5382 format %{ "[$reg, $off]" %}
5383 interface(MEMORY_INTER) %{
5384 base($reg);
5385 index(0xffffffff);
5386 scale(0x0);
5387 disp($off);
5388 %}
5389 %}
5390
5391 operand indOffL2(iRegP reg, immLoffset2 off)
5392 %{
5393 constraint(ALLOC_IN_RC(ptr_reg));
5394 match(AddP reg off);
5395 op_cost(0);
5396 format %{ "[$reg, $off]" %}
5397 interface(MEMORY_INTER) %{
5398 base($reg);
5399 index(0xffffffff);
5400 scale(0x0);
5401 disp($off);
5402 %}
5403 %}
5404
5405 operand indOffL4(iRegP reg, immLoffset4 off)
5406 %{
5407 constraint(ALLOC_IN_RC(ptr_reg));
5408 match(AddP reg off);
5409 op_cost(0);
5410 format %{ "[$reg, $off]" %}
5411 interface(MEMORY_INTER) %{
5412 base($reg);
5413 index(0xffffffff);
5414 scale(0x0);
5415 disp($off);
5416 %}
5417 %}
5418
5419 operand indOffL8(iRegP reg, immLoffset8 off)
5420 %{
5421 constraint(ALLOC_IN_RC(ptr_reg));
5422 match(AddP reg off);
5423 op_cost(0);
5424 format %{ "[$reg, $off]" %}
5425 interface(MEMORY_INTER) %{
5426 base($reg);
5427 index(0xffffffff);
5428 scale(0x0);
5429 disp($off);
5430 %}
5431 %}
5432
5433 operand indOffL16(iRegP reg, immLoffset16 off)
5434 %{
5435 constraint(ALLOC_IN_RC(ptr_reg));
5436 match(AddP reg off);
5437 op_cost(0);
5438 format %{ "[$reg, $off]" %}
5439 interface(MEMORY_INTER) %{
5440 base($reg);
5441 index(0xffffffff);
5442 scale(0x0);
5443 disp($off);
5444 %}
5445 %}
5446
5447 operand indirectX2P(iRegL reg)
5448 %{
5449 constraint(ALLOC_IN_RC(ptr_reg));
5450 match(CastX2P reg);
5451 op_cost(0);
5452 format %{ "[$reg]\t# long -> ptr" %}
5453 interface(MEMORY_INTER) %{
5454 base($reg);
5455 index(0xffffffff);
5456 scale(0x0);
5457 disp(0x0);
5458 %}
5459 %}
5460
5461 operand indOffX2P(iRegL reg, immLOffset off)
5462 %{
5463 constraint(ALLOC_IN_RC(ptr_reg));
5464 match(AddP (CastX2P reg) off);
5465 op_cost(0);
5466 format %{ "[$reg, $off]\t# long -> ptr" %}
5467 interface(MEMORY_INTER) %{
5468 base($reg);
5469 index(0xffffffff);
5470 scale(0x0);
5471 disp($off);
5472 %}
5473 %}
5474
5475 operand indirectN(iRegN reg)
5476 %{
5477 predicate(CompressedOops::shift() == 0);
5478 constraint(ALLOC_IN_RC(ptr_reg));
5479 match(DecodeN reg);
5480 op_cost(0);
5481 format %{ "[$reg]\t# narrow" %}
5482 interface(MEMORY_INTER) %{
5483 base($reg);
5484 index(0xffffffff);
5485 scale(0x0);
5486 disp(0x0);
5487 %}
5488 %}
5489
5490 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5491 %{
5492 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5493 constraint(ALLOC_IN_RC(ptr_reg));
5494 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5495 op_cost(0);
5496 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5497 interface(MEMORY_INTER) %{
5498 base($reg);
5499 index($ireg);
5500 scale($scale);
5501 disp(0x0);
5502 %}
5503 %}
5504
5505 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5506 %{
5507 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5508 constraint(ALLOC_IN_RC(ptr_reg));
5509 match(AddP (DecodeN reg) (LShiftL lreg scale));
5510 op_cost(0);
5511 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5512 interface(MEMORY_INTER) %{
5513 base($reg);
5514 index($lreg);
5515 scale($scale);
5516 disp(0x0);
5517 %}
5518 %}
5519
5520 operand indIndexI2LN(iRegN reg, iRegI ireg)
5521 %{
5522 predicate(CompressedOops::shift() == 0);
5523 constraint(ALLOC_IN_RC(ptr_reg));
5524 match(AddP (DecodeN reg) (ConvI2L ireg));
5525 op_cost(0);
5526 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5527 interface(MEMORY_INTER) %{
5528 base($reg);
5529 index($ireg);
5530 scale(0x0);
5531 disp(0x0);
5532 %}
5533 %}
5534
5535 operand indIndexN(iRegN reg, iRegL lreg)
5536 %{
5537 predicate(CompressedOops::shift() == 0);
5538 constraint(ALLOC_IN_RC(ptr_reg));
5539 match(AddP (DecodeN reg) lreg);
5540 op_cost(0);
5541 format %{ "$reg, $lreg\t# narrow" %}
5542 interface(MEMORY_INTER) %{
5543 base($reg);
5544 index($lreg);
5545 scale(0x0);
5546 disp(0x0);
5547 %}
5548 %}
5549
5550 operand indOffIN(iRegN reg, immIOffset off)
5551 %{
5552 predicate(CompressedOops::shift() == 0);
5553 constraint(ALLOC_IN_RC(ptr_reg));
5554 match(AddP (DecodeN reg) off);
5555 op_cost(0);
5556 format %{ "[$reg, $off]\t# narrow" %}
5557 interface(MEMORY_INTER) %{
5558 base($reg);
5559 index(0xffffffff);
5560 scale(0x0);
5561 disp($off);
5562 %}
5563 %}
5564
5565 operand indOffLN(iRegN reg, immLOffset off)
5566 %{
5567 predicate(CompressedOops::shift() == 0);
5568 constraint(ALLOC_IN_RC(ptr_reg));
5569 match(AddP (DecodeN reg) off);
5570 op_cost(0);
5571 format %{ "[$reg, $off]\t# narrow" %}
5572 interface(MEMORY_INTER) %{
5573 base($reg);
5574 index(0xffffffff);
5575 scale(0x0);
5576 disp($off);
5577 %}
5578 %}
5579
5580
5581 //----------Special Memory Operands--------------------------------------------
5582 // Stack Slot Operand - This operand is used for loading and storing temporary
5583 // values on the stack where a match requires a value to
5584 // flow through memory.
5585 operand stackSlotP(sRegP reg)
5586 %{
5587 constraint(ALLOC_IN_RC(stack_slots));
5588 op_cost(100);
5589 // No match rule because this operand is only generated in matching
5590 // match(RegP);
5591 format %{ "[$reg]" %}
5592 interface(MEMORY_INTER) %{
5593 base(0x1e); // RSP
5594 index(0x0); // No Index
5595 scale(0x0); // No Scale
5596 disp($reg); // Stack Offset
5597 %}
5598 %}
5599
5600 operand stackSlotI(sRegI reg)
5601 %{
5602 constraint(ALLOC_IN_RC(stack_slots));
5603 // No match rule because this operand is only generated in matching
5604 // match(RegI);
5605 format %{ "[$reg]" %}
5606 interface(MEMORY_INTER) %{
5607 base(0x1e); // RSP
5608 index(0x0); // No Index
5609 scale(0x0); // No Scale
5610 disp($reg); // Stack Offset
5611 %}
5612 %}
5613
5614 operand stackSlotF(sRegF reg)
5615 %{
5616 constraint(ALLOC_IN_RC(stack_slots));
5617 // No match rule because this operand is only generated in matching
5618 // match(RegF);
5619 format %{ "[$reg]" %}
5620 interface(MEMORY_INTER) %{
5621 base(0x1e); // RSP
5622 index(0x0); // No Index
5623 scale(0x0); // No Scale
5624 disp($reg); // Stack Offset
5625 %}
5626 %}
5627
5628 operand stackSlotD(sRegD reg)
5629 %{
5630 constraint(ALLOC_IN_RC(stack_slots));
5631 // No match rule because this operand is only generated in matching
5632 // match(RegD);
5633 format %{ "[$reg]" %}
5634 interface(MEMORY_INTER) %{
5635 base(0x1e); // RSP
5636 index(0x0); // No Index
5637 scale(0x0); // No Scale
5638 disp($reg); // Stack Offset
5639 %}
5640 %}
5641
5642 operand stackSlotL(sRegL reg)
5643 %{
5644 constraint(ALLOC_IN_RC(stack_slots));
5645 // No match rule because this operand is only generated in matching
5646 // match(RegL);
5647 format %{ "[$reg]" %}
5648 interface(MEMORY_INTER) %{
5649 base(0x1e); // RSP
5650 index(0x0); // No Index
5651 scale(0x0); // No Scale
5652 disp($reg); // Stack Offset
5653 %}
5654 %}
5655
5656 // Operands for expressing Control Flow
5657 // NOTE: Label is a predefined operand which should not be redefined in
5658 // the AD file. It is generically handled within the ADLC.
5659
5660 //----------Conditional Branch Operands----------------------------------------
5661 // Comparison Op - This is the operation of the comparison, and is limited to
5662 // the following set of codes:
5663 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5664 //
5665 // Other attributes of the comparison, such as unsignedness, are specified
5666 // by the comparison instruction that sets a condition code flags register.
5667 // That result is represented by a flags operand whose subtype is appropriate
5668 // to the unsignedness (etc.) of the comparison.
5669 //
5670 // Later, the instruction which matches both the Comparison Op (a Bool) and
5671 // the flags (produced by the Cmp) specifies the coding of the comparison op
5672 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5673
5674 // used for signed integral comparisons and fp comparisons
5675
5676 operand cmpOp()
5677 %{
5678 match(Bool);
5679
5680 format %{ "" %}
5681 interface(COND_INTER) %{
5682 equal(0x0, "eq");
5683 not_equal(0x1, "ne");
5684 less(0xb, "lt");
5685 greater_equal(0xa, "ge");
5686 less_equal(0xd, "le");
5687 greater(0xc, "gt");
5688 overflow(0x6, "vs");
5689 no_overflow(0x7, "vc");
5690 %}
5691 %}
5692
5693 // used for unsigned integral comparisons
5694
5695 operand cmpOpU()
5696 %{
5697 match(Bool);
5698
5699 format %{ "" %}
5700 interface(COND_INTER) %{
5701 equal(0x0, "eq");
5702 not_equal(0x1, "ne");
5703 less(0x3, "lo");
5704 greater_equal(0x2, "hs");
5705 less_equal(0x9, "ls");
5706 greater(0x8, "hi");
5707 overflow(0x6, "vs");
5708 no_overflow(0x7, "vc");
5709 %}
5710 %}
5711
5712 // used for certain integral comparisons which can be
5713 // converted to cbxx or tbxx instructions
5714
5715 operand cmpOpEqNe()
5716 %{
5717 match(Bool);
5718 op_cost(0);
5719 predicate(n->as_Bool()->_test._test == BoolTest::ne
5720 || n->as_Bool()->_test._test == BoolTest::eq);
5721
5722 format %{ "" %}
5723 interface(COND_INTER) %{
5724 equal(0x0, "eq");
5725 not_equal(0x1, "ne");
5726 less(0xb, "lt");
5727 greater_equal(0xa, "ge");
5728 less_equal(0xd, "le");
5729 greater(0xc, "gt");
5730 overflow(0x6, "vs");
5731 no_overflow(0x7, "vc");
5732 %}
5733 %}
5734
5735 // used for certain integral comparisons which can be
5736 // converted to cbxx or tbxx instructions
5737
5738 operand cmpOpLtGe()
5739 %{
5740 match(Bool);
5741 op_cost(0);
5742
5743 predicate(n->as_Bool()->_test._test == BoolTest::lt
5744 || n->as_Bool()->_test._test == BoolTest::ge);
5745
5746 format %{ "" %}
5747 interface(COND_INTER) %{
5748 equal(0x0, "eq");
5749 not_equal(0x1, "ne");
5750 less(0xb, "lt");
5751 greater_equal(0xa, "ge");
5752 less_equal(0xd, "le");
5753 greater(0xc, "gt");
5754 overflow(0x6, "vs");
5755 no_overflow(0x7, "vc");
5756 %}
5757 %}
5758
5759 // used for certain unsigned integral comparisons which can be
5760 // converted to cbxx or tbxx instructions
5761
5762 operand cmpOpUEqNeLeGt()
5763 %{
5764 match(Bool);
5765 op_cost(0);
5766
5767 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5768 n->as_Bool()->_test._test == BoolTest::ne ||
5769 n->as_Bool()->_test._test == BoolTest::le ||
5770 n->as_Bool()->_test._test == BoolTest::gt);
5771
5772 format %{ "" %}
5773 interface(COND_INTER) %{
5774 equal(0x0, "eq");
5775 not_equal(0x1, "ne");
5776 less(0x3, "lo");
5777 greater_equal(0x2, "hs");
5778 less_equal(0x9, "ls");
5779 greater(0x8, "hi");
5780 overflow(0x6, "vs");
5781 no_overflow(0x7, "vc");
5782 %}
5783 %}
5784
5785 // Special operand allowing long args to int ops to be truncated for free
5786
5787 operand iRegL2I(iRegL reg) %{
5788
5789 op_cost(0);
5790
5791 match(ConvL2I reg);
5792
5793 format %{ "l2i($reg)" %}
5794
5795 interface(REG_INTER)
5796 %}
5797
5798 operand iRegL2P(iRegL reg) %{
5799
5800 op_cost(0);
5801
5802 match(CastX2P reg);
5803
5804 format %{ "l2p($reg)" %}
5805
5806 interface(REG_INTER)
5807 %}
5808
5809 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5810 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5811 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5812 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5813
5814 //----------OPERAND CLASSES----------------------------------------------------
5815 // Operand Classes are groups of operands that are used as to simplify
5816 // instruction definitions by not requiring the AD writer to specify
5817 // separate instructions for every form of operand when the
5818 // instruction accepts multiple operand types with the same basic
5819 // encoding and format. The classic case of this is memory operands.
5820
5821 // memory is used to define read/write location for load/store
5822 // instruction defs. we can turn a memory op into an Address
5823
5824 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5825 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5826
5827 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5828 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5829
5830 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5831 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5832
5833 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5834 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5835
5836 // All of the memory operands. For the pipeline description.
5837 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5838 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5839 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5840
5841
5842 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5843 // operations. it allows the src to be either an iRegI or a (ConvL2I
5844 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5845 // can be elided because the 32-bit instruction will just employ the
5846 // lower 32 bits anyway.
5847 //
5848 // n.b. this does not elide all L2I conversions. if the truncated
5849 // value is consumed by more than one operation then the ConvL2I
5850 // cannot be bundled into the consuming nodes so an l2i gets planted
5851 // (actually a movw $dst $src) and the downstream instructions consume
5852 // the result of the l2i as an iRegI input. That's a shame since the
5853 // movw is actually redundant but its not too costly.
5854
5855 opclass iRegIorL2I(iRegI, iRegL2I);
5856 opclass iRegPorL2P(iRegP, iRegL2P);
5857
5858 //----------PIPELINE-----------------------------------------------------------
5859 // Rules which define the behavior of the target architectures pipeline.
5860
5861 // For specific pipelines, eg A53, define the stages of that pipeline
5862 //pipe_desc(ISS, EX1, EX2, WR);
5863 #define ISS S0
5864 #define EX1 S1
5865 #define EX2 S2
5866 #define WR S3
5867
5868 // Integer ALU reg operation
5869 pipeline %{
5870
5871 attributes %{
5872 // ARM instructions are of fixed length
5873 fixed_size_instructions; // Fixed size instructions TODO does
5874 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5875 // ARM instructions come in 32-bit word units
5876 instruction_unit_size = 4; // An instruction is 4 bytes long
5877 instruction_fetch_unit_size = 64; // The processor fetches one line
5878 instruction_fetch_units = 1; // of 64 bytes
5879 %}
5880
5881 // We don't use an actual pipeline model so don't care about resources
5882 // or description. we do use pipeline classes to introduce fixed
5883 // latencies
5884
5885 //----------RESOURCES----------------------------------------------------------
5886 // Resources are the functional units available to the machine
5887
5888 resources( INS0, INS1, INS01 = INS0 | INS1,
5889 ALU0, ALU1, ALU = ALU0 | ALU1,
5890 MAC,
5891 DIV,
5892 BRANCH,
5893 LDST,
5894 NEON_FP);
5895
5896 //----------PIPELINE DESCRIPTION-----------------------------------------------
5897 // Pipeline Description specifies the stages in the machine's pipeline
5898
5899 // Define the pipeline as a generic 6 stage pipeline
5900 pipe_desc(S0, S1, S2, S3, S4, S5);
5901
5902 //----------PIPELINE CLASSES---------------------------------------------------
5903 // Pipeline Classes describe the stages in which input and output are
5904 // referenced by the hardware pipeline.
5905
5906 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5907 %{
5908 single_instruction;
5909 src1 : S1(read);
5910 src2 : S2(read);
5911 dst : S5(write);
5912 INS01 : ISS;
5913 NEON_FP : S5;
5914 %}
5915
5916 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5917 %{
5918 single_instruction;
5919 src1 : S1(read);
5920 src2 : S2(read);
5921 dst : S5(write);
5922 INS01 : ISS;
5923 NEON_FP : S5;
5924 %}
5925
5926 pipe_class fp_uop_s(vRegF dst, vRegF src)
5927 %{
5928 single_instruction;
5929 src : S1(read);
5930 dst : S5(write);
5931 INS01 : ISS;
5932 NEON_FP : S5;
5933 %}
5934
5935 pipe_class fp_uop_d(vRegD dst, vRegD src)
5936 %{
5937 single_instruction;
5938 src : S1(read);
5939 dst : S5(write);
5940 INS01 : ISS;
5941 NEON_FP : S5;
5942 %}
5943
5944 pipe_class fp_d2f(vRegF dst, vRegD src)
5945 %{
5946 single_instruction;
5947 src : S1(read);
5948 dst : S5(write);
5949 INS01 : ISS;
5950 NEON_FP : S5;
5951 %}
5952
5953 pipe_class fp_f2d(vRegD dst, vRegF src)
5954 %{
5955 single_instruction;
5956 src : S1(read);
5957 dst : S5(write);
5958 INS01 : ISS;
5959 NEON_FP : S5;
5960 %}
5961
5962 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5963 %{
5964 single_instruction;
5965 src : S1(read);
5966 dst : S5(write);
5967 INS01 : ISS;
5968 NEON_FP : S5;
5969 %}
5970
5971 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5972 %{
5973 single_instruction;
5974 src : S1(read);
5975 dst : S5(write);
5976 INS01 : ISS;
5977 NEON_FP : S5;
5978 %}
5979
5980 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5981 %{
5982 single_instruction;
5983 src : S1(read);
5984 dst : S5(write);
5985 INS01 : ISS;
5986 NEON_FP : S5;
5987 %}
5988
5989 pipe_class fp_l2f(vRegF dst, iRegL src)
5990 %{
5991 single_instruction;
5992 src : S1(read);
5993 dst : S5(write);
5994 INS01 : ISS;
5995 NEON_FP : S5;
5996 %}
5997
5998 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5999 %{
6000 single_instruction;
6001 src : S1(read);
6002 dst : S5(write);
6003 INS01 : ISS;
6004 NEON_FP : S5;
6005 %}
6006
6007 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6008 %{
6009 single_instruction;
6010 src : S1(read);
6011 dst : S5(write);
6012 INS01 : ISS;
6013 NEON_FP : S5;
6014 %}
6015
6016 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6017 %{
6018 single_instruction;
6019 src : S1(read);
6020 dst : S5(write);
6021 INS01 : ISS;
6022 NEON_FP : S5;
6023 %}
6024
6025 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6026 %{
6027 single_instruction;
6028 src : S1(read);
6029 dst : S5(write);
6030 INS01 : ISS;
6031 NEON_FP : S5;
6032 %}
6033
6034 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6035 %{
6036 single_instruction;
6037 src1 : S1(read);
6038 src2 : S2(read);
6039 dst : S5(write);
6040 INS0 : ISS;
6041 NEON_FP : S5;
6042 %}
6043
6044 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6045 %{
6046 single_instruction;
6047 src1 : S1(read);
6048 src2 : S2(read);
6049 dst : S5(write);
6050 INS0 : ISS;
6051 NEON_FP : S5;
6052 %}
6053
6054 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6055 %{
6056 single_instruction;
6057 cr : S1(read);
6058 src1 : S1(read);
6059 src2 : S1(read);
6060 dst : S3(write);
6061 INS01 : ISS;
6062 NEON_FP : S3;
6063 %}
6064
6065 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6066 %{
6067 single_instruction;
6068 cr : S1(read);
6069 src1 : S1(read);
6070 src2 : S1(read);
6071 dst : S3(write);
6072 INS01 : ISS;
6073 NEON_FP : S3;
6074 %}
6075
6076 pipe_class fp_imm_s(vRegF dst)
6077 %{
6078 single_instruction;
6079 dst : S3(write);
6080 INS01 : ISS;
6081 NEON_FP : S3;
6082 %}
6083
6084 pipe_class fp_imm_d(vRegD dst)
6085 %{
6086 single_instruction;
6087 dst : S3(write);
6088 INS01 : ISS;
6089 NEON_FP : S3;
6090 %}
6091
6092 pipe_class fp_load_constant_s(vRegF dst)
6093 %{
6094 single_instruction;
6095 dst : S4(write);
6096 INS01 : ISS;
6097 NEON_FP : S4;
6098 %}
6099
6100 pipe_class fp_load_constant_d(vRegD dst)
6101 %{
6102 single_instruction;
6103 dst : S4(write);
6104 INS01 : ISS;
6105 NEON_FP : S4;
6106 %}
6107
6108 //------- Integer ALU operations --------------------------
6109
6110 // Integer ALU reg-reg operation
6111 // Operands needed in EX1, result generated in EX2
6112 // Eg. ADD x0, x1, x2
6113 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6114 %{
6115 single_instruction;
6116 dst : EX2(write);
6117 src1 : EX1(read);
6118 src2 : EX1(read);
6119 INS01 : ISS; // Dual issue as instruction 0 or 1
6120 ALU : EX2;
6121 %}
6122
6123 // Integer ALU reg-reg operation with constant shift
6124 // Shifted register must be available in LATE_ISS instead of EX1
6125 // Eg. ADD x0, x1, x2, LSL #2
6126 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6127 %{
6128 single_instruction;
6129 dst : EX2(write);
6130 src1 : EX1(read);
6131 src2 : ISS(read);
6132 INS01 : ISS;
6133 ALU : EX2;
6134 %}
6135
6136 // Integer ALU reg operation with constant shift
6137 // Eg. LSL x0, x1, #shift
6138 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6139 %{
6140 single_instruction;
6141 dst : EX2(write);
6142 src1 : ISS(read);
6143 INS01 : ISS;
6144 ALU : EX2;
6145 %}
6146
6147 // Integer ALU reg-reg operation with variable shift
6148 // Both operands must be available in LATE_ISS instead of EX1
6149 // Result is available in EX1 instead of EX2
6150 // Eg. LSLV x0, x1, x2
6151 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6152 %{
6153 single_instruction;
6154 dst : EX1(write);
6155 src1 : ISS(read);
6156 src2 : ISS(read);
6157 INS01 : ISS;
6158 ALU : EX1;
6159 %}
6160
6161 // Integer ALU reg-reg operation with extract
6162 // As for _vshift above, but result generated in EX2
6163 // Eg. EXTR x0, x1, x2, #N
6164 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6165 %{
6166 single_instruction;
6167 dst : EX2(write);
6168 src1 : ISS(read);
6169 src2 : ISS(read);
6170 INS1 : ISS; // Can only dual issue as Instruction 1
6171 ALU : EX1;
6172 %}
6173
6174 // Integer ALU reg operation
6175 // Eg. NEG x0, x1
6176 pipe_class ialu_reg(iRegI dst, iRegI src)
6177 %{
6178 single_instruction;
6179 dst : EX2(write);
6180 src : EX1(read);
6181 INS01 : ISS;
6182 ALU : EX2;
6183 %}
6184
6185 // Integer ALU reg mmediate operation
6186 // Eg. ADD x0, x1, #N
6187 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6188 %{
6189 single_instruction;
6190 dst : EX2(write);
6191 src1 : EX1(read);
6192 INS01 : ISS;
6193 ALU : EX2;
6194 %}
6195
6196 // Integer ALU immediate operation (no source operands)
6197 // Eg. MOV x0, #N
6198 pipe_class ialu_imm(iRegI dst)
6199 %{
6200 single_instruction;
6201 dst : EX1(write);
6202 INS01 : ISS;
6203 ALU : EX1;
6204 %}
6205
6206 //------- Compare operation -------------------------------
6207
6208 // Compare reg-reg
6209 // Eg. CMP x0, x1
6210 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6211 %{
6212 single_instruction;
6213 // fixed_latency(16);
6214 cr : EX2(write);
6215 op1 : EX1(read);
6216 op2 : EX1(read);
6217 INS01 : ISS;
6218 ALU : EX2;
6219 %}
6220
6221 // Compare reg-reg
6222 // Eg. CMP x0, #N
6223 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6224 %{
6225 single_instruction;
6226 // fixed_latency(16);
6227 cr : EX2(write);
6228 op1 : EX1(read);
6229 INS01 : ISS;
6230 ALU : EX2;
6231 %}
6232
6233 //------- Conditional instructions ------------------------
6234
6235 // Conditional no operands
6236 // Eg. CSINC x0, zr, zr, <cond>
6237 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6238 %{
6239 single_instruction;
6240 cr : EX1(read);
6241 dst : EX2(write);
6242 INS01 : ISS;
6243 ALU : EX2;
6244 %}
6245
6246 // Conditional 2 operand
6247 // EG. CSEL X0, X1, X2, <cond>
6248 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6249 %{
6250 single_instruction;
6251 cr : EX1(read);
6252 src1 : EX1(read);
6253 src2 : EX1(read);
6254 dst : EX2(write);
6255 INS01 : ISS;
6256 ALU : EX2;
6257 %}
6258
6259 // Conditional 2 operand
6260 // EG. CSEL X0, X1, X2, <cond>
6261 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6262 %{
6263 single_instruction;
6264 cr : EX1(read);
6265 src : EX1(read);
6266 dst : EX2(write);
6267 INS01 : ISS;
6268 ALU : EX2;
6269 %}
6270
6271 //------- Multiply pipeline operations --------------------
6272
6273 // Multiply reg-reg
6274 // Eg. MUL w0, w1, w2
6275 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6276 %{
6277 single_instruction;
6278 dst : WR(write);
6279 src1 : ISS(read);
6280 src2 : ISS(read);
6281 INS01 : ISS;
6282 MAC : WR;
6283 %}
6284
6285 // Multiply accumulate
6286 // Eg. MADD w0, w1, w2, w3
6287 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6288 %{
6289 single_instruction;
6290 dst : WR(write);
6291 src1 : ISS(read);
6292 src2 : ISS(read);
6293 src3 : ISS(read);
6294 INS01 : ISS;
6295 MAC : WR;
6296 %}
6297
6298 // Eg. MUL w0, w1, w2
6299 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6300 %{
6301 single_instruction;
6302 fixed_latency(3); // Maximum latency for 64 bit mul
6303 dst : WR(write);
6304 src1 : ISS(read);
6305 src2 : ISS(read);
6306 INS01 : ISS;
6307 MAC : WR;
6308 %}
6309
6310 // Multiply accumulate
6311 // Eg. MADD w0, w1, w2, w3
6312 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6313 %{
6314 single_instruction;
6315 fixed_latency(3); // Maximum latency for 64 bit mul
6316 dst : WR(write);
6317 src1 : ISS(read);
6318 src2 : ISS(read);
6319 src3 : ISS(read);
6320 INS01 : ISS;
6321 MAC : WR;
6322 %}
6323
6324 //------- Divide pipeline operations --------------------
6325
6326 // Eg. SDIV w0, w1, w2
6327 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6328 %{
6329 single_instruction;
6330 fixed_latency(8); // Maximum latency for 32 bit divide
6331 dst : WR(write);
6332 src1 : ISS(read);
6333 src2 : ISS(read);
6334 INS0 : ISS; // Can only dual issue as instruction 0
6335 DIV : WR;
6336 %}
6337
6338 // Eg. SDIV x0, x1, x2
6339 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6340 %{
6341 single_instruction;
6342 fixed_latency(16); // Maximum latency for 64 bit divide
6343 dst : WR(write);
6344 src1 : ISS(read);
6345 src2 : ISS(read);
6346 INS0 : ISS; // Can only dual issue as instruction 0
6347 DIV : WR;
6348 %}
6349
6350 //------- Load pipeline operations ------------------------
6351
6352 // Load - prefetch
6353 // Eg. PFRM <mem>
6354 pipe_class iload_prefetch(memory mem)
6355 %{
6356 single_instruction;
6357 mem : ISS(read);
6358 INS01 : ISS;
6359 LDST : WR;
6360 %}
6361
6362 // Load - reg, mem
6363 // Eg. LDR x0, <mem>
6364 pipe_class iload_reg_mem(iRegI dst, memory mem)
6365 %{
6366 single_instruction;
6367 dst : WR(write);
6368 mem : ISS(read);
6369 INS01 : ISS;
6370 LDST : WR;
6371 %}
6372
6373 // Load - reg, reg
6374 // Eg. LDR x0, [sp, x1]
6375 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6376 %{
6377 single_instruction;
6378 dst : WR(write);
6379 src : ISS(read);
6380 INS01 : ISS;
6381 LDST : WR;
6382 %}
6383
6384 //------- Store pipeline operations -----------------------
6385
6386 // Store - zr, mem
6387 // Eg. STR zr, <mem>
6388 pipe_class istore_mem(memory mem)
6389 %{
6390 single_instruction;
6391 mem : ISS(read);
6392 INS01 : ISS;
6393 LDST : WR;
6394 %}
6395
6396 // Store - reg, mem
6397 // Eg. STR x0, <mem>
6398 pipe_class istore_reg_mem(iRegI src, memory mem)
6399 %{
6400 single_instruction;
6401 mem : ISS(read);
6402 src : EX2(read);
6403 INS01 : ISS;
6404 LDST : WR;
6405 %}
6406
6407 // Store - reg, reg
6408 // Eg. STR x0, [sp, x1]
6409 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6410 %{
6411 single_instruction;
6412 dst : ISS(read);
6413 src : EX2(read);
6414 INS01 : ISS;
6415 LDST : WR;
6416 %}
6417
6418 //------- Store pipeline operations -----------------------
6419
6420 // Branch
6421 pipe_class pipe_branch()
6422 %{
6423 single_instruction;
6424 INS01 : ISS;
6425 BRANCH : EX1;
6426 %}
6427
6428 // Conditional branch
6429 pipe_class pipe_branch_cond(rFlagsReg cr)
6430 %{
6431 single_instruction;
6432 cr : EX1(read);
6433 INS01 : ISS;
6434 BRANCH : EX1;
6435 %}
6436
6437 // Compare & Branch
6438 // EG. CBZ/CBNZ
6439 pipe_class pipe_cmp_branch(iRegI op1)
6440 %{
6441 single_instruction;
6442 op1 : EX1(read);
6443 INS01 : ISS;
6444 BRANCH : EX1;
6445 %}
6446
6447 //------- Synchronisation operations ----------------------
6448
6449 // Any operation requiring serialization.
6450 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6451 pipe_class pipe_serial()
6452 %{
6453 single_instruction;
6454 force_serialization;
6455 fixed_latency(16);
6456 INS01 : ISS(2); // Cannot dual issue with any other instruction
6457 LDST : WR;
6458 %}
6459
6460 // Generic big/slow expanded idiom - also serialized
6461 pipe_class pipe_slow()
6462 %{
6463 instruction_count(10);
6464 multiple_bundles;
6465 force_serialization;
6466 fixed_latency(16);
6467 INS01 : ISS(2); // Cannot dual issue with any other instruction
6468 LDST : WR;
6469 %}
6470
6471 // Empty pipeline class
6472 pipe_class pipe_class_empty()
6473 %{
6474 single_instruction;
6475 fixed_latency(0);
6476 %}
6477
6478 // Default pipeline class.
6479 pipe_class pipe_class_default()
6480 %{
6481 single_instruction;
6482 fixed_latency(2);
6483 %}
6484
6485 // Pipeline class for compares.
6486 pipe_class pipe_class_compare()
6487 %{
6488 single_instruction;
6489 fixed_latency(16);
6490 %}
6491
6492 // Pipeline class for memory operations.
6493 pipe_class pipe_class_memory()
6494 %{
6495 single_instruction;
6496 fixed_latency(16);
6497 %}
6498
6499 // Pipeline class for call.
6500 pipe_class pipe_class_call()
6501 %{
6502 single_instruction;
6503 fixed_latency(100);
6504 %}
6505
6506 // Define the class for the Nop node.
6507 define %{
6508 MachNop = pipe_class_empty;
6509 %}
6510
6511 %}
6512 //----------INSTRUCTIONS-------------------------------------------------------
6513 //
6514 // match -- States which machine-independent subtree may be replaced
6515 // by this instruction.
6516 // ins_cost -- The estimated cost of this instruction is used by instruction
6517 // selection to identify a minimum cost tree of machine
6518 // instructions that matches a tree of machine-independent
6519 // instructions.
6520 // format -- A string providing the disassembly for this instruction.
6521 // The value of an instruction's operand may be inserted
6522 // by referring to it with a '$' prefix.
6523 // opcode -- Three instruction opcodes may be provided. These are referred
6524 // to within an encode class as $primary, $secondary, and $tertiary
6525 // rrspectively. The primary opcode is commonly used to
6526 // indicate the type of machine instruction, while secondary
6527 // and tertiary are often used for prefix options or addressing
6528 // modes.
6529 // ins_encode -- A list of encode classes with parameters. The encode class
6530 // name must have been defined in an 'enc_class' specification
6531 // in the encode section of the architecture description.
6532
6533 // ============================================================================
6534 // Memory (Load/Store) Instructions
6535
6536 // Load Instructions
6537
6538 // Load Byte (8 bit signed)
6539 instruct loadB(iRegINoSp dst, memory1 mem)
6540 %{
6541 match(Set dst (LoadB mem));
6542 predicate(!needs_acquiring_load(n));
6543
6544 ins_cost(4 * INSN_COST);
6545 format %{ "ldrsbw $dst, $mem\t# byte" %}
6546
6547 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6548
6549 ins_pipe(iload_reg_mem);
6550 %}
6551
6552 // Load Byte (8 bit signed) into long
6553 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6554 %{
6555 match(Set dst (ConvI2L (LoadB mem)));
6556 predicate(!needs_acquiring_load(n->in(1)));
6557
6558 ins_cost(4 * INSN_COST);
6559 format %{ "ldrsb $dst, $mem\t# byte" %}
6560
6561 ins_encode(aarch64_enc_ldrsb(dst, mem));
6562
6563 ins_pipe(iload_reg_mem);
6564 %}
6565
6566 // Load Byte (8 bit unsigned)
6567 instruct loadUB(iRegINoSp dst, memory1 mem)
6568 %{
6569 match(Set dst (LoadUB mem));
6570 predicate(!needs_acquiring_load(n));
6571
6572 ins_cost(4 * INSN_COST);
6573 format %{ "ldrbw $dst, $mem\t# byte" %}
6574
6575 ins_encode(aarch64_enc_ldrb(dst, mem));
6576
6577 ins_pipe(iload_reg_mem);
6578 %}
6579
6580 // Load Byte (8 bit unsigned) into long
6581 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6582 %{
6583 match(Set dst (ConvI2L (LoadUB mem)));
6584 predicate(!needs_acquiring_load(n->in(1)));
6585
6586 ins_cost(4 * INSN_COST);
6587 format %{ "ldrb $dst, $mem\t# byte" %}
6588
6589 ins_encode(aarch64_enc_ldrb(dst, mem));
6590
6591 ins_pipe(iload_reg_mem);
6592 %}
6593
6594 // Load Short (16 bit signed)
6595 instruct loadS(iRegINoSp dst, memory2 mem)
6596 %{
6597 match(Set dst (LoadS mem));
6598 predicate(!needs_acquiring_load(n));
6599
6600 ins_cost(4 * INSN_COST);
6601 format %{ "ldrshw $dst, $mem\t# short" %}
6602
6603 ins_encode(aarch64_enc_ldrshw(dst, mem));
6604
6605 ins_pipe(iload_reg_mem);
6606 %}
6607
6608 // Load Short (16 bit signed) into long
6609 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6610 %{
6611 match(Set dst (ConvI2L (LoadS mem)));
6612 predicate(!needs_acquiring_load(n->in(1)));
6613
6614 ins_cost(4 * INSN_COST);
6615 format %{ "ldrsh $dst, $mem\t# short" %}
6616
6617 ins_encode(aarch64_enc_ldrsh(dst, mem));
6618
6619 ins_pipe(iload_reg_mem);
6620 %}
6621
6622 // Load Char (16 bit unsigned)
6623 instruct loadUS(iRegINoSp dst, memory2 mem)
6624 %{
6625 match(Set dst (LoadUS mem));
6626 predicate(!needs_acquiring_load(n));
6627
6628 ins_cost(4 * INSN_COST);
6629 format %{ "ldrh $dst, $mem\t# short" %}
6630
6631 ins_encode(aarch64_enc_ldrh(dst, mem));
6632
6633 ins_pipe(iload_reg_mem);
6634 %}
6635
6636 // Load Short/Char (16 bit unsigned) into long
6637 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6638 %{
6639 match(Set dst (ConvI2L (LoadUS mem)));
6640 predicate(!needs_acquiring_load(n->in(1)));
6641
6642 ins_cost(4 * INSN_COST);
6643 format %{ "ldrh $dst, $mem\t# short" %}
6644
6645 ins_encode(aarch64_enc_ldrh(dst, mem));
6646
6647 ins_pipe(iload_reg_mem);
6648 %}
6649
6650 // Load Integer (32 bit signed)
6651 instruct loadI(iRegINoSp dst, memory4 mem)
6652 %{
6653 match(Set dst (LoadI mem));
6654 predicate(!needs_acquiring_load(n));
6655
6656 ins_cost(4 * INSN_COST);
6657 format %{ "ldrw $dst, $mem\t# int" %}
6658
6659 ins_encode(aarch64_enc_ldrw(dst, mem));
6660
6661 ins_pipe(iload_reg_mem);
6662 %}
6663
6664 // Load Integer (32 bit signed) into long
6665 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6666 %{
6667 match(Set dst (ConvI2L (LoadI mem)));
6668 predicate(!needs_acquiring_load(n->in(1)));
6669
6670 ins_cost(4 * INSN_COST);
6671 format %{ "ldrsw $dst, $mem\t# int" %}
6672
6673 ins_encode(aarch64_enc_ldrsw(dst, mem));
6674
6675 ins_pipe(iload_reg_mem);
6676 %}
6677
6678 // Load Integer (32 bit unsigned) into long
6679 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6680 %{
6681 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6682 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6683
6684 ins_cost(4 * INSN_COST);
6685 format %{ "ldrw $dst, $mem\t# int" %}
6686
6687 ins_encode(aarch64_enc_ldrw(dst, mem));
6688
6689 ins_pipe(iload_reg_mem);
6690 %}
6691
6692 // Load Long (64 bit signed)
6693 instruct loadL(iRegLNoSp dst, memory8 mem)
6694 %{
6695 match(Set dst (LoadL mem));
6696 predicate(!needs_acquiring_load(n));
6697
6698 ins_cost(4 * INSN_COST);
6699 format %{ "ldr $dst, $mem\t# int" %}
6700
6701 ins_encode(aarch64_enc_ldr(dst, mem));
6702
6703 ins_pipe(iload_reg_mem);
6704 %}
6705
6706 // Load Range
6707 instruct loadRange(iRegINoSp dst, memory4 mem)
6708 %{
6709 match(Set dst (LoadRange mem));
6710
6711 ins_cost(4 * INSN_COST);
6712 format %{ "ldrw $dst, $mem\t# range" %}
6713
6714 ins_encode(aarch64_enc_ldrw(dst, mem));
6715
6716 ins_pipe(iload_reg_mem);
6717 %}
6718
6719 // Load Pointer
6720 instruct loadP(iRegPNoSp dst, memory8 mem)
6721 %{
6722 match(Set dst (LoadP mem));
6723 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6724
6725 ins_cost(4 * INSN_COST);
6726 format %{ "ldr $dst, $mem\t# ptr" %}
6727
6728 ins_encode(aarch64_enc_ldr(dst, mem));
6729
6730 ins_pipe(iload_reg_mem);
6731 %}
6732
6733 // Load Compressed Pointer
6734 instruct loadN(iRegNNoSp dst, memory4 mem)
6735 %{
6736 match(Set dst (LoadN mem));
6737 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6738
6739 ins_cost(4 * INSN_COST);
6740 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6741
6742 ins_encode(aarch64_enc_ldrw(dst, mem));
6743
6744 ins_pipe(iload_reg_mem);
6745 %}
6746
6747 // Load Klass Pointer
6748 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6749 %{
6750 match(Set dst (LoadKlass mem));
6751 predicate(!needs_acquiring_load(n));
6752
6753 ins_cost(4 * INSN_COST);
6754 format %{ "ldr $dst, $mem\t# class" %}
6755
6756 ins_encode(aarch64_enc_ldr(dst, mem));
6757
6758 ins_pipe(iload_reg_mem);
6759 %}
6760
6761 // Load Narrow Klass Pointer
6762 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6763 %{
6764 match(Set dst (LoadNKlass mem));
6765 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6766
6767 ins_cost(4 * INSN_COST);
6768 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6769
6770 ins_encode(aarch64_enc_ldrw(dst, mem));
6771
6772 ins_pipe(iload_reg_mem);
6773 %}
6774
6775 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6776 %{
6777 match(Set dst (LoadNKlass mem));
6778 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6779
6780 ins_cost(4 * INSN_COST);
6781 format %{
6782 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6783 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6784 %}
6785 ins_encode %{
6786 // inlined aarch64_enc_ldrw
6787 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6788 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6789 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6790 %}
6791 ins_pipe(iload_reg_mem);
6792 %}
6793
6794 // Load Float
6795 instruct loadF(vRegF dst, memory4 mem)
6796 %{
6797 match(Set dst (LoadF mem));
6798 predicate(!needs_acquiring_load(n));
6799
6800 ins_cost(4 * INSN_COST);
6801 format %{ "ldrs $dst, $mem\t# float" %}
6802
6803 ins_encode( aarch64_enc_ldrs(dst, mem) );
6804
6805 ins_pipe(pipe_class_memory);
6806 %}
6807
6808 // Load Double
6809 instruct loadD(vRegD dst, memory8 mem)
6810 %{
6811 match(Set dst (LoadD mem));
6812 predicate(!needs_acquiring_load(n));
6813
6814 ins_cost(4 * INSN_COST);
6815 format %{ "ldrd $dst, $mem\t# double" %}
6816
6817 ins_encode( aarch64_enc_ldrd(dst, mem) );
6818
6819 ins_pipe(pipe_class_memory);
6820 %}
6821
6822
6823 // Load Int Constant
6824 instruct loadConI(iRegINoSp dst, immI src)
6825 %{
6826 match(Set dst src);
6827
6828 ins_cost(INSN_COST);
6829 format %{ "mov $dst, $src\t# int" %}
6830
6831 ins_encode( aarch64_enc_movw_imm(dst, src) );
6832
6833 ins_pipe(ialu_imm);
6834 %}
6835
6836 // Load Long Constant
6837 instruct loadConL(iRegLNoSp dst, immL src)
6838 %{
6839 match(Set dst src);
6840
6841 ins_cost(INSN_COST);
6842 format %{ "mov $dst, $src\t# long" %}
6843
6844 ins_encode( aarch64_enc_mov_imm(dst, src) );
6845
6846 ins_pipe(ialu_imm);
6847 %}
6848
6849 // Load Pointer Constant
6850
6851 instruct loadConP(iRegPNoSp dst, immP con)
6852 %{
6853 match(Set dst con);
6854
6855 ins_cost(INSN_COST * 4);
6856 format %{
6857 "mov $dst, $con\t# ptr\n\t"
6858 %}
6859
6860 ins_encode(aarch64_enc_mov_p(dst, con));
6861
6862 ins_pipe(ialu_imm);
6863 %}
6864
6865 // Load Null Pointer Constant
6866
6867 instruct loadConP0(iRegPNoSp dst, immP0 con)
6868 %{
6869 match(Set dst con);
6870
6871 ins_cost(INSN_COST);
6872 format %{ "mov $dst, $con\t# nullptr ptr" %}
6873
6874 ins_encode(aarch64_enc_mov_p0(dst, con));
6875
6876 ins_pipe(ialu_imm);
6877 %}
6878
6879 // Load Pointer Constant One
6880
6881 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6882 %{
6883 match(Set dst con);
6884
6885 ins_cost(INSN_COST);
6886 format %{ "mov $dst, $con\t# nullptr ptr" %}
6887
6888 ins_encode(aarch64_enc_mov_p1(dst, con));
6889
6890 ins_pipe(ialu_imm);
6891 %}
6892
6893 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6894 %{
6895 match(Set dst con);
6896
6897 ins_cost(INSN_COST);
6898 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6899
6900 ins_encode %{
6901 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6902 %}
6903
6904 ins_pipe(ialu_imm);
6905 %}
6906
6907 // Load Narrow Pointer Constant
6908
6909 instruct loadConN(iRegNNoSp dst, immN con)
6910 %{
6911 match(Set dst con);
6912
6913 ins_cost(INSN_COST * 4);
6914 format %{ "mov $dst, $con\t# compressed ptr" %}
6915
6916 ins_encode(aarch64_enc_mov_n(dst, con));
6917
6918 ins_pipe(ialu_imm);
6919 %}
6920
6921 // Load Narrow Null Pointer Constant
6922
6923 instruct loadConN0(iRegNNoSp dst, immN0 con)
6924 %{
6925 match(Set dst con);
6926
6927 ins_cost(INSN_COST);
6928 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6929
6930 ins_encode(aarch64_enc_mov_n0(dst, con));
6931
6932 ins_pipe(ialu_imm);
6933 %}
6934
6935 // Load Narrow Klass Constant
6936
6937 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6938 %{
6939 match(Set dst con);
6940
6941 ins_cost(INSN_COST);
6942 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6943
6944 ins_encode(aarch64_enc_mov_nk(dst, con));
6945
6946 ins_pipe(ialu_imm);
6947 %}
6948
6949 // Load Packed Float Constant
6950
6951 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6952 match(Set dst con);
6953 ins_cost(INSN_COST * 4);
6954 format %{ "fmovs $dst, $con"%}
6955 ins_encode %{
6956 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6957 %}
6958
6959 ins_pipe(fp_imm_s);
6960 %}
6961
6962 // Load Float Constant
6963
6964 instruct loadConF(vRegF dst, immF con) %{
6965 match(Set dst con);
6966
6967 ins_cost(INSN_COST * 4);
6968
6969 format %{
6970 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6971 %}
6972
6973 ins_encode %{
6974 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6975 %}
6976
6977 ins_pipe(fp_load_constant_s);
6978 %}
6979
6980 // Load Packed Double Constant
6981
6982 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6983 match(Set dst con);
6984 ins_cost(INSN_COST);
6985 format %{ "fmovd $dst, $con"%}
6986 ins_encode %{
6987 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6988 %}
6989
6990 ins_pipe(fp_imm_d);
6991 %}
6992
6993 // Load Double Constant
6994
6995 instruct loadConD(vRegD dst, immD con) %{
6996 match(Set dst con);
6997
6998 ins_cost(INSN_COST * 5);
6999 format %{
7000 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7001 %}
7002
7003 ins_encode %{
7004 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7005 %}
7006
7007 ins_pipe(fp_load_constant_d);
7008 %}
7009
7010 // Load Half Float Constant
7011 instruct loadConH(vRegF dst, immH con) %{
7012 match(Set dst con);
7013 format %{ "mov rscratch1, $con\n\t"
7014 "fmov $dst, rscratch1"
7015 %}
7016 ins_encode %{
7017 __ movw(rscratch1, (uint32_t)$con$$constant);
7018 __ fmovs($dst$$FloatRegister, rscratch1);
7019 %}
7020 ins_pipe(pipe_class_default);
7021 %}
7022
7023 // Store Instructions
7024
7025 // Store Byte
7026 instruct storeB(iRegIorL2I src, memory1 mem)
7027 %{
7028 match(Set mem (StoreB mem src));
7029 predicate(!needs_releasing_store(n));
7030
7031 ins_cost(INSN_COST);
7032 format %{ "strb $src, $mem\t# byte" %}
7033
7034 ins_encode(aarch64_enc_strb(src, mem));
7035
7036 ins_pipe(istore_reg_mem);
7037 %}
7038
7039
7040 instruct storeimmB0(immI0 zero, memory1 mem)
7041 %{
7042 match(Set mem (StoreB mem zero));
7043 predicate(!needs_releasing_store(n));
7044
7045 ins_cost(INSN_COST);
7046 format %{ "strb rscractch2, $mem\t# byte" %}
7047
7048 ins_encode(aarch64_enc_strb0(mem));
7049
7050 ins_pipe(istore_mem);
7051 %}
7052
7053 // Store Char/Short
7054 instruct storeC(iRegIorL2I src, memory2 mem)
7055 %{
7056 match(Set mem (StoreC mem src));
7057 predicate(!needs_releasing_store(n));
7058
7059 ins_cost(INSN_COST);
7060 format %{ "strh $src, $mem\t# short" %}
7061
7062 ins_encode(aarch64_enc_strh(src, mem));
7063
7064 ins_pipe(istore_reg_mem);
7065 %}
7066
7067 instruct storeimmC0(immI0 zero, memory2 mem)
7068 %{
7069 match(Set mem (StoreC mem zero));
7070 predicate(!needs_releasing_store(n));
7071
7072 ins_cost(INSN_COST);
7073 format %{ "strh zr, $mem\t# short" %}
7074
7075 ins_encode(aarch64_enc_strh0(mem));
7076
7077 ins_pipe(istore_mem);
7078 %}
7079
7080 // Store Integer
7081
7082 instruct storeI(iRegIorL2I src, memory4 mem)
7083 %{
7084 match(Set mem(StoreI mem src));
7085 predicate(!needs_releasing_store(n));
7086
7087 ins_cost(INSN_COST);
7088 format %{ "strw $src, $mem\t# int" %}
7089
7090 ins_encode(aarch64_enc_strw(src, mem));
7091
7092 ins_pipe(istore_reg_mem);
7093 %}
7094
7095 instruct storeimmI0(immI0 zero, memory4 mem)
7096 %{
7097 match(Set mem(StoreI mem zero));
7098 predicate(!needs_releasing_store(n));
7099
7100 ins_cost(INSN_COST);
7101 format %{ "strw zr, $mem\t# int" %}
7102
7103 ins_encode(aarch64_enc_strw0(mem));
7104
7105 ins_pipe(istore_mem);
7106 %}
7107
7108 // Store Long (64 bit signed)
7109 instruct storeL(iRegL src, memory8 mem)
7110 %{
7111 match(Set mem (StoreL mem src));
7112 predicate(!needs_releasing_store(n));
7113
7114 ins_cost(INSN_COST);
7115 format %{ "str $src, $mem\t# int" %}
7116
7117 ins_encode(aarch64_enc_str(src, mem));
7118
7119 ins_pipe(istore_reg_mem);
7120 %}
7121
7122 // Store Long (64 bit signed)
7123 instruct storeimmL0(immL0 zero, memory8 mem)
7124 %{
7125 match(Set mem (StoreL mem zero));
7126 predicate(!needs_releasing_store(n));
7127
7128 ins_cost(INSN_COST);
7129 format %{ "str zr, $mem\t# int" %}
7130
7131 ins_encode(aarch64_enc_str0(mem));
7132
7133 ins_pipe(istore_mem);
7134 %}
7135
7136 // Store Pointer
7137 instruct storeP(iRegP src, memory8 mem)
7138 %{
7139 match(Set mem (StoreP mem src));
7140 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7141
7142 ins_cost(INSN_COST);
7143 format %{ "str $src, $mem\t# ptr" %}
7144
7145 ins_encode(aarch64_enc_str(src, mem));
7146
7147 ins_pipe(istore_reg_mem);
7148 %}
7149
7150 // Store Pointer
7151 instruct storeimmP0(immP0 zero, memory8 mem)
7152 %{
7153 match(Set mem (StoreP mem zero));
7154 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7155
7156 ins_cost(INSN_COST);
7157 format %{ "str zr, $mem\t# ptr" %}
7158
7159 ins_encode(aarch64_enc_str0(mem));
7160
7161 ins_pipe(istore_mem);
7162 %}
7163
7164 // Store Compressed Pointer
7165 instruct storeN(iRegN src, memory4 mem)
7166 %{
7167 match(Set mem (StoreN mem src));
7168 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7169
7170 ins_cost(INSN_COST);
7171 format %{ "strw $src, $mem\t# compressed ptr" %}
7172
7173 ins_encode(aarch64_enc_strw(src, mem));
7174
7175 ins_pipe(istore_reg_mem);
7176 %}
7177
7178 instruct storeImmN0(immN0 zero, memory4 mem)
7179 %{
7180 match(Set mem (StoreN mem zero));
7181 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7182
7183 ins_cost(INSN_COST);
7184 format %{ "strw zr, $mem\t# compressed ptr" %}
7185
7186 ins_encode(aarch64_enc_strw0(mem));
7187
7188 ins_pipe(istore_mem);
7189 %}
7190
7191 // Store Float
7192 instruct storeF(vRegF src, memory4 mem)
7193 %{
7194 match(Set mem (StoreF mem src));
7195 predicate(!needs_releasing_store(n));
7196
7197 ins_cost(INSN_COST);
7198 format %{ "strs $src, $mem\t# float" %}
7199
7200 ins_encode( aarch64_enc_strs(src, mem) );
7201
7202 ins_pipe(pipe_class_memory);
7203 %}
7204
7205 // TODO
7206 // implement storeImmF0 and storeFImmPacked
7207
7208 // Store Double
7209 instruct storeD(vRegD src, memory8 mem)
7210 %{
7211 match(Set mem (StoreD mem src));
7212 predicate(!needs_releasing_store(n));
7213
7214 ins_cost(INSN_COST);
7215 format %{ "strd $src, $mem\t# double" %}
7216
7217 ins_encode( aarch64_enc_strd(src, mem) );
7218
7219 ins_pipe(pipe_class_memory);
7220 %}
7221
7222 // Store Compressed Klass Pointer
7223 instruct storeNKlass(iRegN src, memory4 mem)
7224 %{
7225 predicate(!needs_releasing_store(n));
7226 match(Set mem (StoreNKlass mem src));
7227
7228 ins_cost(INSN_COST);
7229 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7230
7231 ins_encode(aarch64_enc_strw(src, mem));
7232
7233 ins_pipe(istore_reg_mem);
7234 %}
7235
7236 // TODO
7237 // implement storeImmD0 and storeDImmPacked
7238
7239 // prefetch instructions
7240 // Must be safe to execute with invalid address (cannot fault).
7241
7242 instruct prefetchalloc( memory8 mem ) %{
7243 match(PrefetchAllocation mem);
7244
7245 ins_cost(INSN_COST);
7246 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7247
7248 ins_encode( aarch64_enc_prefetchw(mem) );
7249
7250 ins_pipe(iload_prefetch);
7251 %}
7252
7253 // ---------------- volatile loads and stores ----------------
7254
7255 // Load Byte (8 bit signed)
7256 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7257 %{
7258 match(Set dst (LoadB mem));
7259
7260 ins_cost(VOLATILE_REF_COST);
7261 format %{ "ldarsb $dst, $mem\t# byte" %}
7262
7263 ins_encode(aarch64_enc_ldarsb(dst, mem));
7264
7265 ins_pipe(pipe_serial);
7266 %}
7267
7268 // Load Byte (8 bit signed) into long
7269 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7270 %{
7271 match(Set dst (ConvI2L (LoadB mem)));
7272
7273 ins_cost(VOLATILE_REF_COST);
7274 format %{ "ldarsb $dst, $mem\t# byte" %}
7275
7276 ins_encode(aarch64_enc_ldarsb(dst, mem));
7277
7278 ins_pipe(pipe_serial);
7279 %}
7280
7281 // Load Byte (8 bit unsigned)
7282 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7283 %{
7284 match(Set dst (LoadUB mem));
7285
7286 ins_cost(VOLATILE_REF_COST);
7287 format %{ "ldarb $dst, $mem\t# byte" %}
7288
7289 ins_encode(aarch64_enc_ldarb(dst, mem));
7290
7291 ins_pipe(pipe_serial);
7292 %}
7293
7294 // Load Byte (8 bit unsigned) into long
7295 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7296 %{
7297 match(Set dst (ConvI2L (LoadUB mem)));
7298
7299 ins_cost(VOLATILE_REF_COST);
7300 format %{ "ldarb $dst, $mem\t# byte" %}
7301
7302 ins_encode(aarch64_enc_ldarb(dst, mem));
7303
7304 ins_pipe(pipe_serial);
7305 %}
7306
7307 // Load Short (16 bit signed)
7308 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7309 %{
7310 match(Set dst (LoadS mem));
7311
7312 ins_cost(VOLATILE_REF_COST);
7313 format %{ "ldarshw $dst, $mem\t# short" %}
7314
7315 ins_encode(aarch64_enc_ldarshw(dst, mem));
7316
7317 ins_pipe(pipe_serial);
7318 %}
7319
7320 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7321 %{
7322 match(Set dst (LoadUS mem));
7323
7324 ins_cost(VOLATILE_REF_COST);
7325 format %{ "ldarhw $dst, $mem\t# short" %}
7326
7327 ins_encode(aarch64_enc_ldarhw(dst, mem));
7328
7329 ins_pipe(pipe_serial);
7330 %}
7331
7332 // Load Short/Char (16 bit unsigned) into long
7333 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7334 %{
7335 match(Set dst (ConvI2L (LoadUS mem)));
7336
7337 ins_cost(VOLATILE_REF_COST);
7338 format %{ "ldarh $dst, $mem\t# short" %}
7339
7340 ins_encode(aarch64_enc_ldarh(dst, mem));
7341
7342 ins_pipe(pipe_serial);
7343 %}
7344
7345 // Load Short/Char (16 bit signed) into long
7346 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7347 %{
7348 match(Set dst (ConvI2L (LoadS mem)));
7349
7350 ins_cost(VOLATILE_REF_COST);
7351 format %{ "ldarh $dst, $mem\t# short" %}
7352
7353 ins_encode(aarch64_enc_ldarsh(dst, mem));
7354
7355 ins_pipe(pipe_serial);
7356 %}
7357
7358 // Load Integer (32 bit signed)
7359 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7360 %{
7361 match(Set dst (LoadI mem));
7362
7363 ins_cost(VOLATILE_REF_COST);
7364 format %{ "ldarw $dst, $mem\t# int" %}
7365
7366 ins_encode(aarch64_enc_ldarw(dst, mem));
7367
7368 ins_pipe(pipe_serial);
7369 %}
7370
7371 // Load Integer (32 bit unsigned) into long
7372 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7373 %{
7374 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7375
7376 ins_cost(VOLATILE_REF_COST);
7377 format %{ "ldarw $dst, $mem\t# int" %}
7378
7379 ins_encode(aarch64_enc_ldarw(dst, mem));
7380
7381 ins_pipe(pipe_serial);
7382 %}
7383
7384 // Load Long (64 bit signed)
7385 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7386 %{
7387 match(Set dst (LoadL mem));
7388
7389 ins_cost(VOLATILE_REF_COST);
7390 format %{ "ldar $dst, $mem\t# int" %}
7391
7392 ins_encode(aarch64_enc_ldar(dst, mem));
7393
7394 ins_pipe(pipe_serial);
7395 %}
7396
7397 // Load Pointer
7398 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7399 %{
7400 match(Set dst (LoadP mem));
7401 predicate(n->as_Load()->barrier_data() == 0);
7402
7403 ins_cost(VOLATILE_REF_COST);
7404 format %{ "ldar $dst, $mem\t# ptr" %}
7405
7406 ins_encode(aarch64_enc_ldar(dst, mem));
7407
7408 ins_pipe(pipe_serial);
7409 %}
7410
7411 // Load Compressed Pointer
7412 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7413 %{
7414 match(Set dst (LoadN mem));
7415 predicate(n->as_Load()->barrier_data() == 0);
7416
7417 ins_cost(VOLATILE_REF_COST);
7418 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7419
7420 ins_encode(aarch64_enc_ldarw(dst, mem));
7421
7422 ins_pipe(pipe_serial);
7423 %}
7424
7425 // Load Float
7426 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7427 %{
7428 match(Set dst (LoadF mem));
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "ldars $dst, $mem\t# float" %}
7432
7433 ins_encode( aarch64_enc_fldars(dst, mem) );
7434
7435 ins_pipe(pipe_serial);
7436 %}
7437
7438 // Load Double
7439 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7440 %{
7441 match(Set dst (LoadD mem));
7442
7443 ins_cost(VOLATILE_REF_COST);
7444 format %{ "ldard $dst, $mem\t# double" %}
7445
7446 ins_encode( aarch64_enc_fldard(dst, mem) );
7447
7448 ins_pipe(pipe_serial);
7449 %}
7450
7451 // Store Byte
7452 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set mem (StoreB mem src));
7455
7456 ins_cost(VOLATILE_REF_COST);
7457 format %{ "stlrb $src, $mem\t# byte" %}
7458
7459 ins_encode(aarch64_enc_stlrb(src, mem));
7460
7461 ins_pipe(pipe_class_memory);
7462 %}
7463
7464 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7465 %{
7466 match(Set mem (StoreB mem zero));
7467
7468 ins_cost(VOLATILE_REF_COST);
7469 format %{ "stlrb zr, $mem\t# byte" %}
7470
7471 ins_encode(aarch64_enc_stlrb0(mem));
7472
7473 ins_pipe(pipe_class_memory);
7474 %}
7475
7476 // Store Char/Short
7477 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set mem (StoreC mem src));
7480
7481 ins_cost(VOLATILE_REF_COST);
7482 format %{ "stlrh $src, $mem\t# short" %}
7483
7484 ins_encode(aarch64_enc_stlrh(src, mem));
7485
7486 ins_pipe(pipe_class_memory);
7487 %}
7488
7489 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7490 %{
7491 match(Set mem (StoreC mem zero));
7492
7493 ins_cost(VOLATILE_REF_COST);
7494 format %{ "stlrh zr, $mem\t# short" %}
7495
7496 ins_encode(aarch64_enc_stlrh0(mem));
7497
7498 ins_pipe(pipe_class_memory);
7499 %}
7500
7501 // Store Integer
7502
7503 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7504 %{
7505 match(Set mem(StoreI mem src));
7506
7507 ins_cost(VOLATILE_REF_COST);
7508 format %{ "stlrw $src, $mem\t# int" %}
7509
7510 ins_encode(aarch64_enc_stlrw(src, mem));
7511
7512 ins_pipe(pipe_class_memory);
7513 %}
7514
7515 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7516 %{
7517 match(Set mem(StoreI mem zero));
7518
7519 ins_cost(VOLATILE_REF_COST);
7520 format %{ "stlrw zr, $mem\t# int" %}
7521
7522 ins_encode(aarch64_enc_stlrw0(mem));
7523
7524 ins_pipe(pipe_class_memory);
7525 %}
7526
7527 // Store Long (64 bit signed)
7528 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7529 %{
7530 match(Set mem (StoreL mem src));
7531
7532 ins_cost(VOLATILE_REF_COST);
7533 format %{ "stlr $src, $mem\t# int" %}
7534
7535 ins_encode(aarch64_enc_stlr(src, mem));
7536
7537 ins_pipe(pipe_class_memory);
7538 %}
7539
7540 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7541 %{
7542 match(Set mem (StoreL mem zero));
7543
7544 ins_cost(VOLATILE_REF_COST);
7545 format %{ "stlr zr, $mem\t# int" %}
7546
7547 ins_encode(aarch64_enc_stlr0(mem));
7548
7549 ins_pipe(pipe_class_memory);
7550 %}
7551
7552 // Store Pointer
7553 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7554 %{
7555 match(Set mem (StoreP mem src));
7556 predicate(n->as_Store()->barrier_data() == 0);
7557
7558 ins_cost(VOLATILE_REF_COST);
7559 format %{ "stlr $src, $mem\t# ptr" %}
7560
7561 ins_encode(aarch64_enc_stlr(src, mem));
7562
7563 ins_pipe(pipe_class_memory);
7564 %}
7565
7566 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7567 %{
7568 match(Set mem (StoreP mem zero));
7569 predicate(n->as_Store()->barrier_data() == 0);
7570
7571 ins_cost(VOLATILE_REF_COST);
7572 format %{ "stlr zr, $mem\t# ptr" %}
7573
7574 ins_encode(aarch64_enc_stlr0(mem));
7575
7576 ins_pipe(pipe_class_memory);
7577 %}
7578
7579 // Store Compressed Pointer
7580 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7581 %{
7582 match(Set mem (StoreN mem src));
7583 predicate(n->as_Store()->barrier_data() == 0);
7584
7585 ins_cost(VOLATILE_REF_COST);
7586 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7587
7588 ins_encode(aarch64_enc_stlrw(src, mem));
7589
7590 ins_pipe(pipe_class_memory);
7591 %}
7592
7593 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7594 %{
7595 match(Set mem (StoreN mem zero));
7596 predicate(n->as_Store()->barrier_data() == 0);
7597
7598 ins_cost(VOLATILE_REF_COST);
7599 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7600
7601 ins_encode(aarch64_enc_stlrw0(mem));
7602
7603 ins_pipe(pipe_class_memory);
7604 %}
7605
7606 // Store Float
7607 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7608 %{
7609 match(Set mem (StoreF mem src));
7610
7611 ins_cost(VOLATILE_REF_COST);
7612 format %{ "stlrs $src, $mem\t# float" %}
7613
7614 ins_encode( aarch64_enc_fstlrs(src, mem) );
7615
7616 ins_pipe(pipe_class_memory);
7617 %}
7618
7619 // TODO
7620 // implement storeImmF0 and storeFImmPacked
7621
7622 // Store Double
7623 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7624 %{
7625 match(Set mem (StoreD mem src));
7626
7627 ins_cost(VOLATILE_REF_COST);
7628 format %{ "stlrd $src, $mem\t# double" %}
7629
7630 ins_encode( aarch64_enc_fstlrd(src, mem) );
7631
7632 ins_pipe(pipe_class_memory);
7633 %}
7634
7635 // ---------------- end of volatile loads and stores ----------------
7636
7637 instruct cacheWB(indirect addr)
7638 %{
7639 predicate(VM_Version::supports_data_cache_line_flush());
7640 match(CacheWB addr);
7641
7642 ins_cost(100);
7643 format %{"cache wb $addr" %}
7644 ins_encode %{
7645 assert($addr->index_position() < 0, "should be");
7646 assert($addr$$disp == 0, "should be");
7647 __ cache_wb(Address($addr$$base$$Register, 0));
7648 %}
7649 ins_pipe(pipe_slow); // XXX
7650 %}
7651
7652 instruct cacheWBPreSync()
7653 %{
7654 predicate(VM_Version::supports_data_cache_line_flush());
7655 match(CacheWBPreSync);
7656
7657 ins_cost(100);
7658 format %{"cache wb presync" %}
7659 ins_encode %{
7660 __ cache_wbsync(true);
7661 %}
7662 ins_pipe(pipe_slow); // XXX
7663 %}
7664
7665 instruct cacheWBPostSync()
7666 %{
7667 predicate(VM_Version::supports_data_cache_line_flush());
7668 match(CacheWBPostSync);
7669
7670 ins_cost(100);
7671 format %{"cache wb postsync" %}
7672 ins_encode %{
7673 __ cache_wbsync(false);
7674 %}
7675 ins_pipe(pipe_slow); // XXX
7676 %}
7677
7678 // ============================================================================
7679 // BSWAP Instructions
7680
7681 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7682 match(Set dst (ReverseBytesI src));
7683
7684 ins_cost(INSN_COST);
7685 format %{ "revw $dst, $src" %}
7686
7687 ins_encode %{
7688 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7689 %}
7690
7691 ins_pipe(ialu_reg);
7692 %}
7693
7694 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7695 match(Set dst (ReverseBytesL src));
7696
7697 ins_cost(INSN_COST);
7698 format %{ "rev $dst, $src" %}
7699
7700 ins_encode %{
7701 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7702 %}
7703
7704 ins_pipe(ialu_reg);
7705 %}
7706
7707 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7708 match(Set dst (ReverseBytesUS src));
7709
7710 ins_cost(INSN_COST);
7711 format %{ "rev16w $dst, $src" %}
7712
7713 ins_encode %{
7714 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7715 %}
7716
7717 ins_pipe(ialu_reg);
7718 %}
7719
7720 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7721 match(Set dst (ReverseBytesS src));
7722
7723 ins_cost(INSN_COST);
7724 format %{ "rev16w $dst, $src\n\t"
7725 "sbfmw $dst, $dst, #0, #15" %}
7726
7727 ins_encode %{
7728 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7729 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7730 %}
7731
7732 ins_pipe(ialu_reg);
7733 %}
7734
7735 // ============================================================================
7736 // Zero Count Instructions
7737
7738 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7739 match(Set dst (CountLeadingZerosI src));
7740
7741 ins_cost(INSN_COST);
7742 format %{ "clzw $dst, $src" %}
7743 ins_encode %{
7744 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7745 %}
7746
7747 ins_pipe(ialu_reg);
7748 %}
7749
7750 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7751 match(Set dst (CountLeadingZerosL src));
7752
7753 ins_cost(INSN_COST);
7754 format %{ "clz $dst, $src" %}
7755 ins_encode %{
7756 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7757 %}
7758
7759 ins_pipe(ialu_reg);
7760 %}
7761
7762 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7763 match(Set dst (CountTrailingZerosI src));
7764
7765 ins_cost(INSN_COST * 2);
7766 format %{ "rbitw $dst, $src\n\t"
7767 "clzw $dst, $dst" %}
7768 ins_encode %{
7769 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7770 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7771 %}
7772
7773 ins_pipe(ialu_reg);
7774 %}
7775
7776 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7777 match(Set dst (CountTrailingZerosL src));
7778
7779 ins_cost(INSN_COST * 2);
7780 format %{ "rbit $dst, $src\n\t"
7781 "clz $dst, $dst" %}
7782 ins_encode %{
7783 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7784 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7785 %}
7786
7787 ins_pipe(ialu_reg);
7788 %}
7789
7790 //---------- Population Count Instructions -------------------------------------
7791 //
7792
7793 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7794 match(Set dst (PopCountI src));
7795 effect(TEMP tmp);
7796 ins_cost(INSN_COST * 13);
7797
7798 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7799 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7800 "addv $tmp, $tmp\t# vector (8B)\n\t"
7801 "mov $dst, $tmp\t# vector (1D)" %}
7802 ins_encode %{
7803 __ fmovs($tmp$$FloatRegister, $src$$Register);
7804 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7805 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7806 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7807 %}
7808
7809 ins_pipe(pipe_class_default);
7810 %}
7811
7812 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7813 match(Set dst (PopCountI (LoadI mem)));
7814 effect(TEMP tmp);
7815 ins_cost(INSN_COST * 13);
7816
7817 format %{ "ldrs $tmp, $mem\n\t"
7818 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7819 "addv $tmp, $tmp\t# vector (8B)\n\t"
7820 "mov $dst, $tmp\t# vector (1D)" %}
7821 ins_encode %{
7822 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7823 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7824 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7825 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7826 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7827 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7828 %}
7829
7830 ins_pipe(pipe_class_default);
7831 %}
7832
7833 // Note: Long.bitCount(long) returns an int.
7834 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7835 match(Set dst (PopCountL src));
7836 effect(TEMP tmp);
7837 ins_cost(INSN_COST * 13);
7838
7839 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7840 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7841 "addv $tmp, $tmp\t# vector (8B)\n\t"
7842 "mov $dst, $tmp\t# vector (1D)" %}
7843 ins_encode %{
7844 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7845 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7846 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7847 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7848 %}
7849
7850 ins_pipe(pipe_class_default);
7851 %}
7852
7853 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7854 match(Set dst (PopCountL (LoadL mem)));
7855 effect(TEMP tmp);
7856 ins_cost(INSN_COST * 13);
7857
7858 format %{ "ldrd $tmp, $mem\n\t"
7859 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7860 "addv $tmp, $tmp\t# vector (8B)\n\t"
7861 "mov $dst, $tmp\t# vector (1D)" %}
7862 ins_encode %{
7863 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7864 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7865 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7866 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7867 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7868 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7869 %}
7870
7871 ins_pipe(pipe_class_default);
7872 %}
7873
7874 // ============================================================================
7875 // VerifyVectorAlignment Instruction
7876
7877 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7878 match(Set addr (VerifyVectorAlignment addr mask));
7879 effect(KILL cr);
7880 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7881 ins_encode %{
7882 Label Lskip;
7883 // check if masked bits of addr are zero
7884 __ tst($addr$$Register, $mask$$constant);
7885 __ br(Assembler::EQ, Lskip);
7886 __ stop("verify_vector_alignment found a misaligned vector memory access");
7887 __ bind(Lskip);
7888 %}
7889 ins_pipe(pipe_slow);
7890 %}
7891
7892 // ============================================================================
7893 // MemBar Instruction
7894
7895 instruct load_fence() %{
7896 match(LoadFence);
7897 ins_cost(VOLATILE_REF_COST);
7898
7899 format %{ "load_fence" %}
7900
7901 ins_encode %{
7902 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7903 %}
7904 ins_pipe(pipe_serial);
7905 %}
7906
7907 instruct unnecessary_membar_acquire() %{
7908 predicate(unnecessary_acquire(n));
7909 match(MemBarAcquire);
7910 ins_cost(0);
7911
7912 format %{ "membar_acquire (elided)" %}
7913
7914 ins_encode %{
7915 __ block_comment("membar_acquire (elided)");
7916 %}
7917
7918 ins_pipe(pipe_class_empty);
7919 %}
7920
7921 instruct membar_acquire() %{
7922 match(MemBarAcquire);
7923 ins_cost(VOLATILE_REF_COST);
7924
7925 format %{ "membar_acquire\n\t"
7926 "dmb ishld" %}
7927
7928 ins_encode %{
7929 __ block_comment("membar_acquire");
7930 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7931 %}
7932
7933 ins_pipe(pipe_serial);
7934 %}
7935
7936
7937 instruct membar_acquire_lock() %{
7938 match(MemBarAcquireLock);
7939 ins_cost(VOLATILE_REF_COST);
7940
7941 format %{ "membar_acquire_lock (elided)" %}
7942
7943 ins_encode %{
7944 __ block_comment("membar_acquire_lock (elided)");
7945 %}
7946
7947 ins_pipe(pipe_serial);
7948 %}
7949
7950 instruct store_fence() %{
7951 match(StoreFence);
7952 ins_cost(VOLATILE_REF_COST);
7953
7954 format %{ "store_fence" %}
7955
7956 ins_encode %{
7957 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7958 %}
7959 ins_pipe(pipe_serial);
7960 %}
7961
7962 instruct unnecessary_membar_release() %{
7963 predicate(unnecessary_release(n));
7964 match(MemBarRelease);
7965 ins_cost(0);
7966
7967 format %{ "membar_release (elided)" %}
7968
7969 ins_encode %{
7970 __ block_comment("membar_release (elided)");
7971 %}
7972 ins_pipe(pipe_serial);
7973 %}
7974
7975 instruct membar_release() %{
7976 match(MemBarRelease);
7977 ins_cost(VOLATILE_REF_COST);
7978
7979 format %{ "membar_release\n\t"
7980 "dmb ishst\n\tdmb ishld" %}
7981
7982 ins_encode %{
7983 __ block_comment("membar_release");
7984 // These will be merged if AlwaysMergeDMB is enabled.
7985 __ membar(Assembler::StoreStore);
7986 __ membar(Assembler::LoadStore);
7987 %}
7988 ins_pipe(pipe_serial);
7989 %}
7990
7991 instruct membar_storestore() %{
7992 match(MemBarStoreStore);
7993 match(StoreStoreFence);
7994 ins_cost(VOLATILE_REF_COST);
7995
7996 format %{ "MEMBAR-store-store" %}
7997
7998 ins_encode %{
7999 __ membar(Assembler::StoreStore);
8000 %}
8001 ins_pipe(pipe_serial);
8002 %}
8003
8004 instruct membar_release_lock() %{
8005 match(MemBarReleaseLock);
8006 ins_cost(VOLATILE_REF_COST);
8007
8008 format %{ "membar_release_lock (elided)" %}
8009
8010 ins_encode %{
8011 __ block_comment("membar_release_lock (elided)");
8012 %}
8013
8014 ins_pipe(pipe_serial);
8015 %}
8016
8017 instruct membar_storeload() %{
8018 match(MemBarStoreLoad);
8019 ins_cost(VOLATILE_REF_COST*100);
8020
8021 format %{ "MEMBAR-store-load\n\t"
8022 "dmb ish" %}
8023
8024 ins_encode %{
8025 __ block_comment("membar_storeload");
8026 __ membar(Assembler::StoreLoad);
8027 %}
8028
8029 ins_pipe(pipe_serial);
8030 %}
8031
8032 instruct unnecessary_membar_volatile() %{
8033 predicate(unnecessary_volatile(n));
8034 match(MemBarVolatile);
8035 ins_cost(0);
8036
8037 format %{ "membar_volatile (elided)" %}
8038
8039 ins_encode %{
8040 __ block_comment("membar_volatile (elided)");
8041 %}
8042
8043 ins_pipe(pipe_serial);
8044 %}
8045
8046 instruct membar_volatile() %{
8047 match(MemBarVolatile);
8048 ins_cost(VOLATILE_REF_COST*100);
8049
8050 format %{ "membar_volatile\n\t"
8051 "dmb ish"%}
8052
8053 ins_encode %{
8054 __ block_comment("membar_volatile");
8055 __ membar(Assembler::StoreLoad);
8056 %}
8057
8058 ins_pipe(pipe_serial);
8059 %}
8060
8061 instruct membar_full() %{
8062 match(MemBarFull);
8063 ins_cost(VOLATILE_REF_COST*100);
8064
8065 format %{ "membar_full\n\t"
8066 "dmb ish" %}
8067 ins_encode %{
8068 __ block_comment("membar_full");
8069 __ membar(Assembler::AnyAny);
8070 %}
8071
8072 ins_pipe(pipe_serial);
8073 %}
8074
8075 // ============================================================================
8076 // Cast/Convert Instructions
8077
8078 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8079 match(Set dst (CastX2P src));
8080
8081 ins_cost(INSN_COST);
8082 format %{ "mov $dst, $src\t# long -> ptr" %}
8083
8084 ins_encode %{
8085 if ($dst$$reg != $src$$reg) {
8086 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8087 }
8088 %}
8089
8090 ins_pipe(ialu_reg);
8091 %}
8092
8093 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8094 match(Set dst (CastP2X src));
8095
8096 ins_cost(INSN_COST);
8097 format %{ "mov $dst, $src\t# ptr -> long" %}
8098
8099 ins_encode %{
8100 if ($dst$$reg != $src$$reg) {
8101 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8102 }
8103 %}
8104
8105 ins_pipe(ialu_reg);
8106 %}
8107
8108 // Convert oop into int for vectors alignment masking
8109 instruct convP2I(iRegINoSp dst, iRegP src) %{
8110 match(Set dst (ConvL2I (CastP2X src)));
8111
8112 ins_cost(INSN_COST);
8113 format %{ "movw $dst, $src\t# ptr -> int" %}
8114 ins_encode %{
8115 __ movw($dst$$Register, $src$$Register);
8116 %}
8117
8118 ins_pipe(ialu_reg);
8119 %}
8120
8121 // Convert compressed oop into int for vectors alignment masking
8122 // in case of 32bit oops (heap < 4Gb).
8123 instruct convN2I(iRegINoSp dst, iRegN src)
8124 %{
8125 predicate(CompressedOops::shift() == 0);
8126 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8127
8128 ins_cost(INSN_COST);
8129 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8130 ins_encode %{
8131 __ movw($dst$$Register, $src$$Register);
8132 %}
8133
8134 ins_pipe(ialu_reg);
8135 %}
8136
8137
8138 // Convert oop pointer into compressed form
8139 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8140 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8141 match(Set dst (EncodeP src));
8142 effect(KILL cr);
8143 ins_cost(INSN_COST * 3);
8144 format %{ "encode_heap_oop $dst, $src" %}
8145 ins_encode %{
8146 Register s = $src$$Register;
8147 Register d = $dst$$Register;
8148 __ encode_heap_oop(d, s);
8149 %}
8150 ins_pipe(ialu_reg);
8151 %}
8152
8153 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8154 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8155 match(Set dst (EncodeP src));
8156 ins_cost(INSN_COST * 3);
8157 format %{ "encode_heap_oop_not_null $dst, $src" %}
8158 ins_encode %{
8159 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8160 %}
8161 ins_pipe(ialu_reg);
8162 %}
8163
8164 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8165 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8166 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8167 match(Set dst (DecodeN src));
8168 ins_cost(INSN_COST * 3);
8169 format %{ "decode_heap_oop $dst, $src" %}
8170 ins_encode %{
8171 Register s = $src$$Register;
8172 Register d = $dst$$Register;
8173 __ decode_heap_oop(d, s);
8174 %}
8175 ins_pipe(ialu_reg);
8176 %}
8177
8178 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8179 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8180 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8181 match(Set dst (DecodeN src));
8182 ins_cost(INSN_COST * 3);
8183 format %{ "decode_heap_oop_not_null $dst, $src" %}
8184 ins_encode %{
8185 Register s = $src$$Register;
8186 Register d = $dst$$Register;
8187 __ decode_heap_oop_not_null(d, s);
8188 %}
8189 ins_pipe(ialu_reg);
8190 %}
8191
8192 // n.b. AArch64 implementations of encode_klass_not_null and
8193 // decode_klass_not_null do not modify the flags register so, unlike
8194 // Intel, we don't kill CR as a side effect here
8195
8196 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8197 match(Set dst (EncodePKlass src));
8198
8199 ins_cost(INSN_COST * 3);
8200 format %{ "encode_klass_not_null $dst,$src" %}
8201
8202 ins_encode %{
8203 Register src_reg = as_Register($src$$reg);
8204 Register dst_reg = as_Register($dst$$reg);
8205 __ encode_klass_not_null(dst_reg, src_reg);
8206 %}
8207
8208 ins_pipe(ialu_reg);
8209 %}
8210
8211 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8212 match(Set dst (DecodeNKlass src));
8213
8214 ins_cost(INSN_COST * 3);
8215 format %{ "decode_klass_not_null $dst,$src" %}
8216
8217 ins_encode %{
8218 Register src_reg = as_Register($src$$reg);
8219 Register dst_reg = as_Register($dst$$reg);
8220 if (dst_reg != src_reg) {
8221 __ decode_klass_not_null(dst_reg, src_reg);
8222 } else {
8223 __ decode_klass_not_null(dst_reg);
8224 }
8225 %}
8226
8227 ins_pipe(ialu_reg);
8228 %}
8229
8230 instruct checkCastPP(iRegPNoSp dst)
8231 %{
8232 match(Set dst (CheckCastPP dst));
8233
8234 size(0);
8235 format %{ "# checkcastPP of $dst" %}
8236 ins_encode(/* empty encoding */);
8237 ins_pipe(pipe_class_empty);
8238 %}
8239
8240 instruct castPP(iRegPNoSp dst)
8241 %{
8242 match(Set dst (CastPP dst));
8243
8244 size(0);
8245 format %{ "# castPP of $dst" %}
8246 ins_encode(/* empty encoding */);
8247 ins_pipe(pipe_class_empty);
8248 %}
8249
8250 instruct castII(iRegI dst)
8251 %{
8252 predicate(VerifyConstraintCasts == 0);
8253 match(Set dst (CastII dst));
8254
8255 size(0);
8256 format %{ "# castII of $dst" %}
8257 ins_encode(/* empty encoding */);
8258 ins_cost(0);
8259 ins_pipe(pipe_class_empty);
8260 %}
8261
8262 instruct castII_checked(iRegI dst, rFlagsReg cr)
8263 %{
8264 predicate(VerifyConstraintCasts > 0);
8265 match(Set dst (CastII dst));
8266 effect(KILL cr);
8267
8268 format %{ "# castII_checked of $dst" %}
8269 ins_encode %{
8270 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8271 %}
8272 ins_pipe(pipe_slow);
8273 %}
8274
8275 instruct castLL(iRegL dst)
8276 %{
8277 predicate(VerifyConstraintCasts == 0);
8278 match(Set dst (CastLL dst));
8279
8280 size(0);
8281 format %{ "# castLL of $dst" %}
8282 ins_encode(/* empty encoding */);
8283 ins_cost(0);
8284 ins_pipe(pipe_class_empty);
8285 %}
8286
8287 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8288 %{
8289 predicate(VerifyConstraintCasts > 0);
8290 match(Set dst (CastLL dst));
8291 effect(KILL cr);
8292
8293 format %{ "# castLL_checked of $dst" %}
8294 ins_encode %{
8295 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8296 %}
8297 ins_pipe(pipe_slow);
8298 %}
8299
8300 instruct castHH(vRegF dst)
8301 %{
8302 match(Set dst (CastHH dst));
8303 size(0);
8304 format %{ "# castHH of $dst" %}
8305 ins_encode(/* empty encoding */);
8306 ins_cost(0);
8307 ins_pipe(pipe_class_empty);
8308 %}
8309
8310 instruct castFF(vRegF dst)
8311 %{
8312 match(Set dst (CastFF dst));
8313
8314 size(0);
8315 format %{ "# castFF of $dst" %}
8316 ins_encode(/* empty encoding */);
8317 ins_cost(0);
8318 ins_pipe(pipe_class_empty);
8319 %}
8320
8321 instruct castDD(vRegD dst)
8322 %{
8323 match(Set dst (CastDD dst));
8324
8325 size(0);
8326 format %{ "# castDD of $dst" %}
8327 ins_encode(/* empty encoding */);
8328 ins_cost(0);
8329 ins_pipe(pipe_class_empty);
8330 %}
8331
8332 instruct castVV(vReg dst)
8333 %{
8334 match(Set dst (CastVV dst));
8335
8336 size(0);
8337 format %{ "# castVV of $dst" %}
8338 ins_encode(/* empty encoding */);
8339 ins_cost(0);
8340 ins_pipe(pipe_class_empty);
8341 %}
8342
8343 instruct castVVMask(pRegGov dst)
8344 %{
8345 match(Set dst (CastVV dst));
8346
8347 size(0);
8348 format %{ "# castVV of $dst" %}
8349 ins_encode(/* empty encoding */);
8350 ins_cost(0);
8351 ins_pipe(pipe_class_empty);
8352 %}
8353
8354 // Manifest a CmpU result in an integer register.
8355 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8356 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8357 %{
8358 match(Set dst (CmpU3 src1 src2));
8359 effect(KILL flags);
8360
8361 ins_cost(INSN_COST * 3);
8362 format %{
8363 "cmpw $src1, $src2\n\t"
8364 "csetw $dst, ne\n\t"
8365 "cnegw $dst, lo\t# CmpU3(reg)"
8366 %}
8367 ins_encode %{
8368 __ cmpw($src1$$Register, $src2$$Register);
8369 __ csetw($dst$$Register, Assembler::NE);
8370 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8371 %}
8372
8373 ins_pipe(pipe_class_default);
8374 %}
8375
8376 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8377 %{
8378 match(Set dst (CmpU3 src1 src2));
8379 effect(KILL flags);
8380
8381 ins_cost(INSN_COST * 3);
8382 format %{
8383 "subsw zr, $src1, $src2\n\t"
8384 "csetw $dst, ne\n\t"
8385 "cnegw $dst, lo\t# CmpU3(imm)"
8386 %}
8387 ins_encode %{
8388 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8389 __ csetw($dst$$Register, Assembler::NE);
8390 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8391 %}
8392
8393 ins_pipe(pipe_class_default);
8394 %}
8395
8396 // Manifest a CmpUL result in an integer register.
8397 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8398 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8399 %{
8400 match(Set dst (CmpUL3 src1 src2));
8401 effect(KILL flags);
8402
8403 ins_cost(INSN_COST * 3);
8404 format %{
8405 "cmp $src1, $src2\n\t"
8406 "csetw $dst, ne\n\t"
8407 "cnegw $dst, lo\t# CmpUL3(reg)"
8408 %}
8409 ins_encode %{
8410 __ cmp($src1$$Register, $src2$$Register);
8411 __ csetw($dst$$Register, Assembler::NE);
8412 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8413 %}
8414
8415 ins_pipe(pipe_class_default);
8416 %}
8417
8418 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8419 %{
8420 match(Set dst (CmpUL3 src1 src2));
8421 effect(KILL flags);
8422
8423 ins_cost(INSN_COST * 3);
8424 format %{
8425 "subs zr, $src1, $src2\n\t"
8426 "csetw $dst, ne\n\t"
8427 "cnegw $dst, lo\t# CmpUL3(imm)"
8428 %}
8429 ins_encode %{
8430 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8431 __ csetw($dst$$Register, Assembler::NE);
8432 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8433 %}
8434
8435 ins_pipe(pipe_class_default);
8436 %}
8437
8438 // Manifest a CmpL result in an integer register.
8439 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8440 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8441 %{
8442 match(Set dst (CmpL3 src1 src2));
8443 effect(KILL flags);
8444
8445 ins_cost(INSN_COST * 3);
8446 format %{
8447 "cmp $src1, $src2\n\t"
8448 "csetw $dst, ne\n\t"
8449 "cnegw $dst, lt\t# CmpL3(reg)"
8450 %}
8451 ins_encode %{
8452 __ cmp($src1$$Register, $src2$$Register);
8453 __ csetw($dst$$Register, Assembler::NE);
8454 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8455 %}
8456
8457 ins_pipe(pipe_class_default);
8458 %}
8459
8460 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8461 %{
8462 match(Set dst (CmpL3 src1 src2));
8463 effect(KILL flags);
8464
8465 ins_cost(INSN_COST * 3);
8466 format %{
8467 "subs zr, $src1, $src2\n\t"
8468 "csetw $dst, ne\n\t"
8469 "cnegw $dst, lt\t# CmpL3(imm)"
8470 %}
8471 ins_encode %{
8472 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8473 __ csetw($dst$$Register, Assembler::NE);
8474 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8475 %}
8476
8477 ins_pipe(pipe_class_default);
8478 %}
8479
8480 // ============================================================================
8481 // Conditional Move Instructions
8482
8483 // n.b. we have identical rules for both a signed compare op (cmpOp)
8484 // and an unsigned compare op (cmpOpU). it would be nice if we could
8485 // define an op class which merged both inputs and use it to type the
8486 // argument to a single rule. unfortunatelyt his fails because the
8487 // opclass does not live up to the COND_INTER interface of its
8488 // component operands. When the generic code tries to negate the
8489 // operand it ends up running the generci Machoper::negate method
8490 // which throws a ShouldNotHappen. So, we have to provide two flavours
8491 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8492
8493 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8494 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8495
8496 ins_cost(INSN_COST * 2);
8497 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8498
8499 ins_encode %{
8500 __ cselw(as_Register($dst$$reg),
8501 as_Register($src2$$reg),
8502 as_Register($src1$$reg),
8503 (Assembler::Condition)$cmp$$cmpcode);
8504 %}
8505
8506 ins_pipe(icond_reg_reg);
8507 %}
8508
8509 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8510 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8511
8512 ins_cost(INSN_COST * 2);
8513 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8514
8515 ins_encode %{
8516 __ cselw(as_Register($dst$$reg),
8517 as_Register($src2$$reg),
8518 as_Register($src1$$reg),
8519 (Assembler::Condition)$cmp$$cmpcode);
8520 %}
8521
8522 ins_pipe(icond_reg_reg);
8523 %}
8524
8525 // special cases where one arg is zero
8526
8527 // n.b. this is selected in preference to the rule above because it
8528 // avoids loading constant 0 into a source register
8529
8530 // TODO
8531 // we ought only to be able to cull one of these variants as the ideal
8532 // transforms ought always to order the zero consistently (to left/right?)
8533
8534 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8535 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8536
8537 ins_cost(INSN_COST * 2);
8538 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8539
8540 ins_encode %{
8541 __ cselw(as_Register($dst$$reg),
8542 as_Register($src$$reg),
8543 zr,
8544 (Assembler::Condition)$cmp$$cmpcode);
8545 %}
8546
8547 ins_pipe(icond_reg);
8548 %}
8549
8550 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8551 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8552
8553 ins_cost(INSN_COST * 2);
8554 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8555
8556 ins_encode %{
8557 __ cselw(as_Register($dst$$reg),
8558 as_Register($src$$reg),
8559 zr,
8560 (Assembler::Condition)$cmp$$cmpcode);
8561 %}
8562
8563 ins_pipe(icond_reg);
8564 %}
8565
8566 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8567 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8568
8569 ins_cost(INSN_COST * 2);
8570 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8571
8572 ins_encode %{
8573 __ cselw(as_Register($dst$$reg),
8574 zr,
8575 as_Register($src$$reg),
8576 (Assembler::Condition)$cmp$$cmpcode);
8577 %}
8578
8579 ins_pipe(icond_reg);
8580 %}
8581
8582 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8583 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8584
8585 ins_cost(INSN_COST * 2);
8586 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8587
8588 ins_encode %{
8589 __ cselw(as_Register($dst$$reg),
8590 zr,
8591 as_Register($src$$reg),
8592 (Assembler::Condition)$cmp$$cmpcode);
8593 %}
8594
8595 ins_pipe(icond_reg);
8596 %}
8597
8598 // special case for creating a boolean 0 or 1
8599
8600 // n.b. this is selected in preference to the rule above because it
8601 // avoids loading constants 0 and 1 into a source register
8602
8603 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8604 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8605
8606 ins_cost(INSN_COST * 2);
8607 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8608
8609 ins_encode %{
8610 // equivalently
8611 // cset(as_Register($dst$$reg),
8612 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8613 __ csincw(as_Register($dst$$reg),
8614 zr,
8615 zr,
8616 (Assembler::Condition)$cmp$$cmpcode);
8617 %}
8618
8619 ins_pipe(icond_none);
8620 %}
8621
8622 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8623 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8624
8625 ins_cost(INSN_COST * 2);
8626 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8627
8628 ins_encode %{
8629 // equivalently
8630 // cset(as_Register($dst$$reg),
8631 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8632 __ csincw(as_Register($dst$$reg),
8633 zr,
8634 zr,
8635 (Assembler::Condition)$cmp$$cmpcode);
8636 %}
8637
8638 ins_pipe(icond_none);
8639 %}
8640
8641 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8642 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8643
8644 ins_cost(INSN_COST * 2);
8645 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8646
8647 ins_encode %{
8648 __ csel(as_Register($dst$$reg),
8649 as_Register($src2$$reg),
8650 as_Register($src1$$reg),
8651 (Assembler::Condition)$cmp$$cmpcode);
8652 %}
8653
8654 ins_pipe(icond_reg_reg);
8655 %}
8656
8657 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8658 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8659
8660 ins_cost(INSN_COST * 2);
8661 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8662
8663 ins_encode %{
8664 __ csel(as_Register($dst$$reg),
8665 as_Register($src2$$reg),
8666 as_Register($src1$$reg),
8667 (Assembler::Condition)$cmp$$cmpcode);
8668 %}
8669
8670 ins_pipe(icond_reg_reg);
8671 %}
8672
8673 // special cases where one arg is zero
8674
8675 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8676 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8677
8678 ins_cost(INSN_COST * 2);
8679 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8680
8681 ins_encode %{
8682 __ csel(as_Register($dst$$reg),
8683 zr,
8684 as_Register($src$$reg),
8685 (Assembler::Condition)$cmp$$cmpcode);
8686 %}
8687
8688 ins_pipe(icond_reg);
8689 %}
8690
8691 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8692 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8693
8694 ins_cost(INSN_COST * 2);
8695 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8696
8697 ins_encode %{
8698 __ csel(as_Register($dst$$reg),
8699 zr,
8700 as_Register($src$$reg),
8701 (Assembler::Condition)$cmp$$cmpcode);
8702 %}
8703
8704 ins_pipe(icond_reg);
8705 %}
8706
8707 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8708 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8709
8710 ins_cost(INSN_COST * 2);
8711 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8712
8713 ins_encode %{
8714 __ csel(as_Register($dst$$reg),
8715 as_Register($src$$reg),
8716 zr,
8717 (Assembler::Condition)$cmp$$cmpcode);
8718 %}
8719
8720 ins_pipe(icond_reg);
8721 %}
8722
8723 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8724 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8725
8726 ins_cost(INSN_COST * 2);
8727 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8728
8729 ins_encode %{
8730 __ csel(as_Register($dst$$reg),
8731 as_Register($src$$reg),
8732 zr,
8733 (Assembler::Condition)$cmp$$cmpcode);
8734 %}
8735
8736 ins_pipe(icond_reg);
8737 %}
8738
8739 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8740 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8741
8742 ins_cost(INSN_COST * 2);
8743 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8744
8745 ins_encode %{
8746 __ csel(as_Register($dst$$reg),
8747 as_Register($src2$$reg),
8748 as_Register($src1$$reg),
8749 (Assembler::Condition)$cmp$$cmpcode);
8750 %}
8751
8752 ins_pipe(icond_reg_reg);
8753 %}
8754
8755 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8756 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8757
8758 ins_cost(INSN_COST * 2);
8759 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8760
8761 ins_encode %{
8762 __ csel(as_Register($dst$$reg),
8763 as_Register($src2$$reg),
8764 as_Register($src1$$reg),
8765 (Assembler::Condition)$cmp$$cmpcode);
8766 %}
8767
8768 ins_pipe(icond_reg_reg);
8769 %}
8770
8771 // special cases where one arg is zero
8772
8773 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8774 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8775
8776 ins_cost(INSN_COST * 2);
8777 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8778
8779 ins_encode %{
8780 __ csel(as_Register($dst$$reg),
8781 zr,
8782 as_Register($src$$reg),
8783 (Assembler::Condition)$cmp$$cmpcode);
8784 %}
8785
8786 ins_pipe(icond_reg);
8787 %}
8788
8789 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8790 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8791
8792 ins_cost(INSN_COST * 2);
8793 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8794
8795 ins_encode %{
8796 __ csel(as_Register($dst$$reg),
8797 zr,
8798 as_Register($src$$reg),
8799 (Assembler::Condition)$cmp$$cmpcode);
8800 %}
8801
8802 ins_pipe(icond_reg);
8803 %}
8804
8805 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8806 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8807
8808 ins_cost(INSN_COST * 2);
8809 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8810
8811 ins_encode %{
8812 __ csel(as_Register($dst$$reg),
8813 as_Register($src$$reg),
8814 zr,
8815 (Assembler::Condition)$cmp$$cmpcode);
8816 %}
8817
8818 ins_pipe(icond_reg);
8819 %}
8820
8821 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8822 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8823
8824 ins_cost(INSN_COST * 2);
8825 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8826
8827 ins_encode %{
8828 __ csel(as_Register($dst$$reg),
8829 as_Register($src$$reg),
8830 zr,
8831 (Assembler::Condition)$cmp$$cmpcode);
8832 %}
8833
8834 ins_pipe(icond_reg);
8835 %}
8836
8837 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8838 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8839
8840 ins_cost(INSN_COST * 2);
8841 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8842
8843 ins_encode %{
8844 __ cselw(as_Register($dst$$reg),
8845 as_Register($src2$$reg),
8846 as_Register($src1$$reg),
8847 (Assembler::Condition)$cmp$$cmpcode);
8848 %}
8849
8850 ins_pipe(icond_reg_reg);
8851 %}
8852
8853 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8854 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8855
8856 ins_cost(INSN_COST * 2);
8857 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8858
8859 ins_encode %{
8860 __ cselw(as_Register($dst$$reg),
8861 as_Register($src2$$reg),
8862 as_Register($src1$$reg),
8863 (Assembler::Condition)$cmp$$cmpcode);
8864 %}
8865
8866 ins_pipe(icond_reg_reg);
8867 %}
8868
8869 // special cases where one arg is zero
8870
8871 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8872 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8873
8874 ins_cost(INSN_COST * 2);
8875 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8876
8877 ins_encode %{
8878 __ cselw(as_Register($dst$$reg),
8879 zr,
8880 as_Register($src$$reg),
8881 (Assembler::Condition)$cmp$$cmpcode);
8882 %}
8883
8884 ins_pipe(icond_reg);
8885 %}
8886
8887 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8888 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8889
8890 ins_cost(INSN_COST * 2);
8891 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8892
8893 ins_encode %{
8894 __ cselw(as_Register($dst$$reg),
8895 zr,
8896 as_Register($src$$reg),
8897 (Assembler::Condition)$cmp$$cmpcode);
8898 %}
8899
8900 ins_pipe(icond_reg);
8901 %}
8902
8903 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8904 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8905
8906 ins_cost(INSN_COST * 2);
8907 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8908
8909 ins_encode %{
8910 __ cselw(as_Register($dst$$reg),
8911 as_Register($src$$reg),
8912 zr,
8913 (Assembler::Condition)$cmp$$cmpcode);
8914 %}
8915
8916 ins_pipe(icond_reg);
8917 %}
8918
8919 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8920 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8921
8922 ins_cost(INSN_COST * 2);
8923 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8924
8925 ins_encode %{
8926 __ cselw(as_Register($dst$$reg),
8927 as_Register($src$$reg),
8928 zr,
8929 (Assembler::Condition)$cmp$$cmpcode);
8930 %}
8931
8932 ins_pipe(icond_reg);
8933 %}
8934
8935 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8936 %{
8937 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8938
8939 ins_cost(INSN_COST * 3);
8940
8941 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8942 ins_encode %{
8943 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8944 __ fcsels(as_FloatRegister($dst$$reg),
8945 as_FloatRegister($src2$$reg),
8946 as_FloatRegister($src1$$reg),
8947 cond);
8948 %}
8949
8950 ins_pipe(fp_cond_reg_reg_s);
8951 %}
8952
8953 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8954 %{
8955 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8956
8957 ins_cost(INSN_COST * 3);
8958
8959 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8960 ins_encode %{
8961 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8962 __ fcsels(as_FloatRegister($dst$$reg),
8963 as_FloatRegister($src2$$reg),
8964 as_FloatRegister($src1$$reg),
8965 cond);
8966 %}
8967
8968 ins_pipe(fp_cond_reg_reg_s);
8969 %}
8970
8971 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8972 %{
8973 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8974
8975 ins_cost(INSN_COST * 3);
8976
8977 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8978 ins_encode %{
8979 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8980 __ fcseld(as_FloatRegister($dst$$reg),
8981 as_FloatRegister($src2$$reg),
8982 as_FloatRegister($src1$$reg),
8983 cond);
8984 %}
8985
8986 ins_pipe(fp_cond_reg_reg_d);
8987 %}
8988
8989 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8990 %{
8991 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8992
8993 ins_cost(INSN_COST * 3);
8994
8995 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8996 ins_encode %{
8997 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8998 __ fcseld(as_FloatRegister($dst$$reg),
8999 as_FloatRegister($src2$$reg),
9000 as_FloatRegister($src1$$reg),
9001 cond);
9002 %}
9003
9004 ins_pipe(fp_cond_reg_reg_d);
9005 %}
9006
9007 // ============================================================================
9008 // Arithmetic Instructions
9009 //
9010
9011 // Integer Addition
9012
9013 // TODO
9014 // these currently employ operations which do not set CR and hence are
9015 // not flagged as killing CR but we would like to isolate the cases
9016 // where we want to set flags from those where we don't. need to work
9017 // out how to do that.
9018
9019 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9020 match(Set dst (AddI src1 src2));
9021
9022 ins_cost(INSN_COST);
9023 format %{ "addw $dst, $src1, $src2" %}
9024
9025 ins_encode %{
9026 __ addw(as_Register($dst$$reg),
9027 as_Register($src1$$reg),
9028 as_Register($src2$$reg));
9029 %}
9030
9031 ins_pipe(ialu_reg_reg);
9032 %}
9033
9034 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9035 match(Set dst (AddI src1 src2));
9036
9037 ins_cost(INSN_COST);
9038 format %{ "addw $dst, $src1, $src2" %}
9039
9040 // use opcode to indicate that this is an add not a sub
9041 opcode(0x0);
9042
9043 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9044
9045 ins_pipe(ialu_reg_imm);
9046 %}
9047
9048 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9049 match(Set dst (AddI (ConvL2I src1) src2));
9050
9051 ins_cost(INSN_COST);
9052 format %{ "addw $dst, $src1, $src2" %}
9053
9054 // use opcode to indicate that this is an add not a sub
9055 opcode(0x0);
9056
9057 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9058
9059 ins_pipe(ialu_reg_imm);
9060 %}
9061
9062 // Pointer Addition
9063 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9064 match(Set dst (AddP src1 src2));
9065
9066 ins_cost(INSN_COST);
9067 format %{ "add $dst, $src1, $src2\t# ptr" %}
9068
9069 ins_encode %{
9070 __ add(as_Register($dst$$reg),
9071 as_Register($src1$$reg),
9072 as_Register($src2$$reg));
9073 %}
9074
9075 ins_pipe(ialu_reg_reg);
9076 %}
9077
9078 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9079 match(Set dst (AddP src1 (ConvI2L src2)));
9080
9081 ins_cost(1.9 * INSN_COST);
9082 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9083
9084 ins_encode %{
9085 __ add(as_Register($dst$$reg),
9086 as_Register($src1$$reg),
9087 as_Register($src2$$reg), ext::sxtw);
9088 %}
9089
9090 ins_pipe(ialu_reg_reg);
9091 %}
9092
9093 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9094 match(Set dst (AddP src1 (LShiftL src2 scale)));
9095
9096 ins_cost(1.9 * INSN_COST);
9097 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9098
9099 ins_encode %{
9100 __ lea(as_Register($dst$$reg),
9101 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9102 Address::lsl($scale$$constant)));
9103 %}
9104
9105 ins_pipe(ialu_reg_reg_shift);
9106 %}
9107
9108 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9109 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9110
9111 ins_cost(1.9 * INSN_COST);
9112 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9113
9114 ins_encode %{
9115 __ lea(as_Register($dst$$reg),
9116 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9117 Address::sxtw($scale$$constant)));
9118 %}
9119
9120 ins_pipe(ialu_reg_reg_shift);
9121 %}
9122
9123 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9124 match(Set dst (LShiftL (ConvI2L src) scale));
9125
9126 ins_cost(INSN_COST);
9127 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9128
9129 ins_encode %{
9130 __ sbfiz(as_Register($dst$$reg),
9131 as_Register($src$$reg),
9132 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9133 %}
9134
9135 ins_pipe(ialu_reg_shift);
9136 %}
9137
9138 // Pointer Immediate Addition
9139 // n.b. this needs to be more expensive than using an indirect memory
9140 // operand
9141 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9142 match(Set dst (AddP src1 src2));
9143
9144 ins_cost(INSN_COST);
9145 format %{ "add $dst, $src1, $src2\t# ptr" %}
9146
9147 // use opcode to indicate that this is an add not a sub
9148 opcode(0x0);
9149
9150 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9151
9152 ins_pipe(ialu_reg_imm);
9153 %}
9154
9155 // Long Addition
9156 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9157
9158 match(Set dst (AddL src1 src2));
9159
9160 ins_cost(INSN_COST);
9161 format %{ "add $dst, $src1, $src2" %}
9162
9163 ins_encode %{
9164 __ add(as_Register($dst$$reg),
9165 as_Register($src1$$reg),
9166 as_Register($src2$$reg));
9167 %}
9168
9169 ins_pipe(ialu_reg_reg);
9170 %}
9171
9172 // No constant pool entries requiredLong Immediate Addition.
9173 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9174 match(Set dst (AddL src1 src2));
9175
9176 ins_cost(INSN_COST);
9177 format %{ "add $dst, $src1, $src2" %}
9178
9179 // use opcode to indicate that this is an add not a sub
9180 opcode(0x0);
9181
9182 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9183
9184 ins_pipe(ialu_reg_imm);
9185 %}
9186
9187 // Integer Subtraction
9188 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9189 match(Set dst (SubI src1 src2));
9190
9191 ins_cost(INSN_COST);
9192 format %{ "subw $dst, $src1, $src2" %}
9193
9194 ins_encode %{
9195 __ subw(as_Register($dst$$reg),
9196 as_Register($src1$$reg),
9197 as_Register($src2$$reg));
9198 %}
9199
9200 ins_pipe(ialu_reg_reg);
9201 %}
9202
9203 // Immediate Subtraction
9204 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9205 match(Set dst (SubI src1 src2));
9206
9207 ins_cost(INSN_COST);
9208 format %{ "subw $dst, $src1, $src2" %}
9209
9210 // use opcode to indicate that this is a sub not an add
9211 opcode(0x1);
9212
9213 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9214
9215 ins_pipe(ialu_reg_imm);
9216 %}
9217
9218 // Long Subtraction
9219 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9220
9221 match(Set dst (SubL src1 src2));
9222
9223 ins_cost(INSN_COST);
9224 format %{ "sub $dst, $src1, $src2" %}
9225
9226 ins_encode %{
9227 __ sub(as_Register($dst$$reg),
9228 as_Register($src1$$reg),
9229 as_Register($src2$$reg));
9230 %}
9231
9232 ins_pipe(ialu_reg_reg);
9233 %}
9234
9235 // No constant pool entries requiredLong Immediate Subtraction.
9236 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9237 match(Set dst (SubL src1 src2));
9238
9239 ins_cost(INSN_COST);
9240 format %{ "sub$dst, $src1, $src2" %}
9241
9242 // use opcode to indicate that this is a sub not an add
9243 opcode(0x1);
9244
9245 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9246
9247 ins_pipe(ialu_reg_imm);
9248 %}
9249
9250 // Integer Negation (special case for sub)
9251
9252 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9253 match(Set dst (SubI zero src));
9254
9255 ins_cost(INSN_COST);
9256 format %{ "negw $dst, $src\t# int" %}
9257
9258 ins_encode %{
9259 __ negw(as_Register($dst$$reg),
9260 as_Register($src$$reg));
9261 %}
9262
9263 ins_pipe(ialu_reg);
9264 %}
9265
9266 // Long Negation
9267
9268 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9269 match(Set dst (SubL zero src));
9270
9271 ins_cost(INSN_COST);
9272 format %{ "neg $dst, $src\t# long" %}
9273
9274 ins_encode %{
9275 __ neg(as_Register($dst$$reg),
9276 as_Register($src$$reg));
9277 %}
9278
9279 ins_pipe(ialu_reg);
9280 %}
9281
9282 // Integer Multiply
9283
9284 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9285 match(Set dst (MulI src1 src2));
9286
9287 ins_cost(INSN_COST * 3);
9288 format %{ "mulw $dst, $src1, $src2" %}
9289
9290 ins_encode %{
9291 __ mulw(as_Register($dst$$reg),
9292 as_Register($src1$$reg),
9293 as_Register($src2$$reg));
9294 %}
9295
9296 ins_pipe(imul_reg_reg);
9297 %}
9298
9299 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9300 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9301
9302 ins_cost(INSN_COST * 3);
9303 format %{ "smull $dst, $src1, $src2" %}
9304
9305 ins_encode %{
9306 __ smull(as_Register($dst$$reg),
9307 as_Register($src1$$reg),
9308 as_Register($src2$$reg));
9309 %}
9310
9311 ins_pipe(imul_reg_reg);
9312 %}
9313
9314 // Long Multiply
9315
9316 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9317 match(Set dst (MulL src1 src2));
9318
9319 ins_cost(INSN_COST * 5);
9320 format %{ "mul $dst, $src1, $src2" %}
9321
9322 ins_encode %{
9323 __ mul(as_Register($dst$$reg),
9324 as_Register($src1$$reg),
9325 as_Register($src2$$reg));
9326 %}
9327
9328 ins_pipe(lmul_reg_reg);
9329 %}
9330
9331 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9332 %{
9333 match(Set dst (MulHiL src1 src2));
9334
9335 ins_cost(INSN_COST * 7);
9336 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9337
9338 ins_encode %{
9339 __ smulh(as_Register($dst$$reg),
9340 as_Register($src1$$reg),
9341 as_Register($src2$$reg));
9342 %}
9343
9344 ins_pipe(lmul_reg_reg);
9345 %}
9346
9347 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9348 %{
9349 match(Set dst (UMulHiL src1 src2));
9350
9351 ins_cost(INSN_COST * 7);
9352 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9353
9354 ins_encode %{
9355 __ umulh(as_Register($dst$$reg),
9356 as_Register($src1$$reg),
9357 as_Register($src2$$reg));
9358 %}
9359
9360 ins_pipe(lmul_reg_reg);
9361 %}
9362
9363 // Combined Integer Multiply & Add/Sub
9364
9365 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9366 match(Set dst (AddI src3 (MulI src1 src2)));
9367
9368 ins_cost(INSN_COST * 3);
9369 format %{ "madd $dst, $src1, $src2, $src3" %}
9370
9371 ins_encode %{
9372 __ maddw(as_Register($dst$$reg),
9373 as_Register($src1$$reg),
9374 as_Register($src2$$reg),
9375 as_Register($src3$$reg));
9376 %}
9377
9378 ins_pipe(imac_reg_reg);
9379 %}
9380
9381 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9382 match(Set dst (SubI src3 (MulI src1 src2)));
9383
9384 ins_cost(INSN_COST * 3);
9385 format %{ "msub $dst, $src1, $src2, $src3" %}
9386
9387 ins_encode %{
9388 __ msubw(as_Register($dst$$reg),
9389 as_Register($src1$$reg),
9390 as_Register($src2$$reg),
9391 as_Register($src3$$reg));
9392 %}
9393
9394 ins_pipe(imac_reg_reg);
9395 %}
9396
9397 // Combined Integer Multiply & Neg
9398
9399 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9400 match(Set dst (MulI (SubI zero src1) src2));
9401
9402 ins_cost(INSN_COST * 3);
9403 format %{ "mneg $dst, $src1, $src2" %}
9404
9405 ins_encode %{
9406 __ mnegw(as_Register($dst$$reg),
9407 as_Register($src1$$reg),
9408 as_Register($src2$$reg));
9409 %}
9410
9411 ins_pipe(imac_reg_reg);
9412 %}
9413
9414 // Combined Long Multiply & Add/Sub
9415
9416 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9417 match(Set dst (AddL src3 (MulL src1 src2)));
9418
9419 ins_cost(INSN_COST * 5);
9420 format %{ "madd $dst, $src1, $src2, $src3" %}
9421
9422 ins_encode %{
9423 __ madd(as_Register($dst$$reg),
9424 as_Register($src1$$reg),
9425 as_Register($src2$$reg),
9426 as_Register($src3$$reg));
9427 %}
9428
9429 ins_pipe(lmac_reg_reg);
9430 %}
9431
9432 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9433 match(Set dst (SubL src3 (MulL src1 src2)));
9434
9435 ins_cost(INSN_COST * 5);
9436 format %{ "msub $dst, $src1, $src2, $src3" %}
9437
9438 ins_encode %{
9439 __ msub(as_Register($dst$$reg),
9440 as_Register($src1$$reg),
9441 as_Register($src2$$reg),
9442 as_Register($src3$$reg));
9443 %}
9444
9445 ins_pipe(lmac_reg_reg);
9446 %}
9447
9448 // Combined Long Multiply & Neg
9449
9450 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9451 match(Set dst (MulL (SubL zero src1) src2));
9452
9453 ins_cost(INSN_COST * 5);
9454 format %{ "mneg $dst, $src1, $src2" %}
9455
9456 ins_encode %{
9457 __ mneg(as_Register($dst$$reg),
9458 as_Register($src1$$reg),
9459 as_Register($src2$$reg));
9460 %}
9461
9462 ins_pipe(lmac_reg_reg);
9463 %}
9464
9465 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9466
9467 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9468 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9469
9470 ins_cost(INSN_COST * 3);
9471 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9472
9473 ins_encode %{
9474 __ smaddl(as_Register($dst$$reg),
9475 as_Register($src1$$reg),
9476 as_Register($src2$$reg),
9477 as_Register($src3$$reg));
9478 %}
9479
9480 ins_pipe(imac_reg_reg);
9481 %}
9482
9483 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9484 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9485
9486 ins_cost(INSN_COST * 3);
9487 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9488
9489 ins_encode %{
9490 __ smsubl(as_Register($dst$$reg),
9491 as_Register($src1$$reg),
9492 as_Register($src2$$reg),
9493 as_Register($src3$$reg));
9494 %}
9495
9496 ins_pipe(imac_reg_reg);
9497 %}
9498
9499 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9500 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9501
9502 ins_cost(INSN_COST * 3);
9503 format %{ "smnegl $dst, $src1, $src2" %}
9504
9505 ins_encode %{
9506 __ smnegl(as_Register($dst$$reg),
9507 as_Register($src1$$reg),
9508 as_Register($src2$$reg));
9509 %}
9510
9511 ins_pipe(imac_reg_reg);
9512 %}
9513
9514 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9515
9516 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9517 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9518
9519 ins_cost(INSN_COST * 5);
9520 format %{ "mulw rscratch1, $src1, $src2\n\t"
9521 "maddw $dst, $src3, $src4, rscratch1" %}
9522
9523 ins_encode %{
9524 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9525 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9526
9527 ins_pipe(imac_reg_reg);
9528 %}
9529
9530 // Integer Divide
9531
9532 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9533 match(Set dst (DivI src1 src2));
9534
9535 ins_cost(INSN_COST * 19);
9536 format %{ "sdivw $dst, $src1, $src2" %}
9537
9538 ins_encode(aarch64_enc_divw(dst, src1, src2));
9539 ins_pipe(idiv_reg_reg);
9540 %}
9541
9542 // Long Divide
9543
9544 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9545 match(Set dst (DivL src1 src2));
9546
9547 ins_cost(INSN_COST * 35);
9548 format %{ "sdiv $dst, $src1, $src2" %}
9549
9550 ins_encode(aarch64_enc_div(dst, src1, src2));
9551 ins_pipe(ldiv_reg_reg);
9552 %}
9553
9554 // Integer Remainder
9555
9556 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9557 match(Set dst (ModI src1 src2));
9558
9559 ins_cost(INSN_COST * 22);
9560 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9561 "msubw $dst, rscratch1, $src2, $src1" %}
9562
9563 ins_encode(aarch64_enc_modw(dst, src1, src2));
9564 ins_pipe(idiv_reg_reg);
9565 %}
9566
9567 // Long Remainder
9568
9569 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9570 match(Set dst (ModL src1 src2));
9571
9572 ins_cost(INSN_COST * 38);
9573 format %{ "sdiv rscratch1, $src1, $src2\n"
9574 "msub $dst, rscratch1, $src2, $src1" %}
9575
9576 ins_encode(aarch64_enc_mod(dst, src1, src2));
9577 ins_pipe(ldiv_reg_reg);
9578 %}
9579
9580 // Unsigned Integer Divide
9581
9582 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9583 match(Set dst (UDivI src1 src2));
9584
9585 ins_cost(INSN_COST * 19);
9586 format %{ "udivw $dst, $src1, $src2" %}
9587
9588 ins_encode %{
9589 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9590 %}
9591
9592 ins_pipe(idiv_reg_reg);
9593 %}
9594
9595 // Unsigned Long Divide
9596
9597 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9598 match(Set dst (UDivL src1 src2));
9599
9600 ins_cost(INSN_COST * 35);
9601 format %{ "udiv $dst, $src1, $src2" %}
9602
9603 ins_encode %{
9604 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9605 %}
9606
9607 ins_pipe(ldiv_reg_reg);
9608 %}
9609
9610 // Unsigned Integer Remainder
9611
9612 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9613 match(Set dst (UModI src1 src2));
9614
9615 ins_cost(INSN_COST * 22);
9616 format %{ "udivw rscratch1, $src1, $src2\n\t"
9617 "msubw $dst, rscratch1, $src2, $src1" %}
9618
9619 ins_encode %{
9620 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9621 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9622 %}
9623
9624 ins_pipe(idiv_reg_reg);
9625 %}
9626
9627 // Unsigned Long Remainder
9628
9629 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9630 match(Set dst (UModL src1 src2));
9631
9632 ins_cost(INSN_COST * 38);
9633 format %{ "udiv rscratch1, $src1, $src2\n"
9634 "msub $dst, rscratch1, $src2, $src1" %}
9635
9636 ins_encode %{
9637 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9638 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9639 %}
9640
9641 ins_pipe(ldiv_reg_reg);
9642 %}
9643
9644 // Integer Shifts
9645
9646 // Shift Left Register
9647 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9648 match(Set dst (LShiftI src1 src2));
9649
9650 ins_cost(INSN_COST * 2);
9651 format %{ "lslvw $dst, $src1, $src2" %}
9652
9653 ins_encode %{
9654 __ lslvw(as_Register($dst$$reg),
9655 as_Register($src1$$reg),
9656 as_Register($src2$$reg));
9657 %}
9658
9659 ins_pipe(ialu_reg_reg_vshift);
9660 %}
9661
9662 // Shift Left Immediate
9663 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9664 match(Set dst (LShiftI src1 src2));
9665
9666 ins_cost(INSN_COST);
9667 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9668
9669 ins_encode %{
9670 __ lslw(as_Register($dst$$reg),
9671 as_Register($src1$$reg),
9672 $src2$$constant & 0x1f);
9673 %}
9674
9675 ins_pipe(ialu_reg_shift);
9676 %}
9677
9678 // Shift Right Logical Register
9679 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9680 match(Set dst (URShiftI src1 src2));
9681
9682 ins_cost(INSN_COST * 2);
9683 format %{ "lsrvw $dst, $src1, $src2" %}
9684
9685 ins_encode %{
9686 __ lsrvw(as_Register($dst$$reg),
9687 as_Register($src1$$reg),
9688 as_Register($src2$$reg));
9689 %}
9690
9691 ins_pipe(ialu_reg_reg_vshift);
9692 %}
9693
9694 // Shift Right Logical Immediate
9695 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9696 match(Set dst (URShiftI src1 src2));
9697
9698 ins_cost(INSN_COST);
9699 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9700
9701 ins_encode %{
9702 __ lsrw(as_Register($dst$$reg),
9703 as_Register($src1$$reg),
9704 $src2$$constant & 0x1f);
9705 %}
9706
9707 ins_pipe(ialu_reg_shift);
9708 %}
9709
9710 // Shift Right Arithmetic Register
9711 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9712 match(Set dst (RShiftI src1 src2));
9713
9714 ins_cost(INSN_COST * 2);
9715 format %{ "asrvw $dst, $src1, $src2" %}
9716
9717 ins_encode %{
9718 __ asrvw(as_Register($dst$$reg),
9719 as_Register($src1$$reg),
9720 as_Register($src2$$reg));
9721 %}
9722
9723 ins_pipe(ialu_reg_reg_vshift);
9724 %}
9725
9726 // Shift Right Arithmetic Immediate
9727 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9728 match(Set dst (RShiftI src1 src2));
9729
9730 ins_cost(INSN_COST);
9731 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9732
9733 ins_encode %{
9734 __ asrw(as_Register($dst$$reg),
9735 as_Register($src1$$reg),
9736 $src2$$constant & 0x1f);
9737 %}
9738
9739 ins_pipe(ialu_reg_shift);
9740 %}
9741
9742 // Combined Int Mask and Right Shift (using UBFM)
9743 // TODO
9744
9745 // Long Shifts
9746
9747 // Shift Left Register
9748 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9749 match(Set dst (LShiftL src1 src2));
9750
9751 ins_cost(INSN_COST * 2);
9752 format %{ "lslv $dst, $src1, $src2" %}
9753
9754 ins_encode %{
9755 __ lslv(as_Register($dst$$reg),
9756 as_Register($src1$$reg),
9757 as_Register($src2$$reg));
9758 %}
9759
9760 ins_pipe(ialu_reg_reg_vshift);
9761 %}
9762
9763 // Shift Left Immediate
9764 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9765 match(Set dst (LShiftL src1 src2));
9766
9767 ins_cost(INSN_COST);
9768 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9769
9770 ins_encode %{
9771 __ lsl(as_Register($dst$$reg),
9772 as_Register($src1$$reg),
9773 $src2$$constant & 0x3f);
9774 %}
9775
9776 ins_pipe(ialu_reg_shift);
9777 %}
9778
9779 // Shift Right Logical Register
9780 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9781 match(Set dst (URShiftL src1 src2));
9782
9783 ins_cost(INSN_COST * 2);
9784 format %{ "lsrv $dst, $src1, $src2" %}
9785
9786 ins_encode %{
9787 __ lsrv(as_Register($dst$$reg),
9788 as_Register($src1$$reg),
9789 as_Register($src2$$reg));
9790 %}
9791
9792 ins_pipe(ialu_reg_reg_vshift);
9793 %}
9794
9795 // Shift Right Logical Immediate
9796 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9797 match(Set dst (URShiftL src1 src2));
9798
9799 ins_cost(INSN_COST);
9800 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9801
9802 ins_encode %{
9803 __ lsr(as_Register($dst$$reg),
9804 as_Register($src1$$reg),
9805 $src2$$constant & 0x3f);
9806 %}
9807
9808 ins_pipe(ialu_reg_shift);
9809 %}
9810
9811 // A special-case pattern for card table stores.
9812 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9813 match(Set dst (URShiftL (CastP2X src1) src2));
9814
9815 ins_cost(INSN_COST);
9816 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9817
9818 ins_encode %{
9819 __ lsr(as_Register($dst$$reg),
9820 as_Register($src1$$reg),
9821 $src2$$constant & 0x3f);
9822 %}
9823
9824 ins_pipe(ialu_reg_shift);
9825 %}
9826
9827 // Shift Right Arithmetic Register
9828 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9829 match(Set dst (RShiftL src1 src2));
9830
9831 ins_cost(INSN_COST * 2);
9832 format %{ "asrv $dst, $src1, $src2" %}
9833
9834 ins_encode %{
9835 __ asrv(as_Register($dst$$reg),
9836 as_Register($src1$$reg),
9837 as_Register($src2$$reg));
9838 %}
9839
9840 ins_pipe(ialu_reg_reg_vshift);
9841 %}
9842
9843 // Shift Right Arithmetic Immediate
9844 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9845 match(Set dst (RShiftL src1 src2));
9846
9847 ins_cost(INSN_COST);
9848 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9849
9850 ins_encode %{
9851 __ asr(as_Register($dst$$reg),
9852 as_Register($src1$$reg),
9853 $src2$$constant & 0x3f);
9854 %}
9855
9856 ins_pipe(ialu_reg_shift);
9857 %}
9858
9859 // BEGIN This section of the file is automatically generated. Do not edit --------------
9860 // This section is generated from aarch64_ad.m4
9861
9862 // This pattern is automatically generated from aarch64_ad.m4
9863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9864 instruct regL_not_reg(iRegLNoSp dst,
9865 iRegL src1, immL_M1 m1,
9866 rFlagsReg cr) %{
9867 match(Set dst (XorL src1 m1));
9868 ins_cost(INSN_COST);
9869 format %{ "eon $dst, $src1, zr" %}
9870
9871 ins_encode %{
9872 __ eon(as_Register($dst$$reg),
9873 as_Register($src1$$reg),
9874 zr,
9875 Assembler::LSL, 0);
9876 %}
9877
9878 ins_pipe(ialu_reg);
9879 %}
9880
9881 // This pattern is automatically generated from aarch64_ad.m4
9882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9883 instruct regI_not_reg(iRegINoSp dst,
9884 iRegIorL2I src1, immI_M1 m1,
9885 rFlagsReg cr) %{
9886 match(Set dst (XorI src1 m1));
9887 ins_cost(INSN_COST);
9888 format %{ "eonw $dst, $src1, zr" %}
9889
9890 ins_encode %{
9891 __ eonw(as_Register($dst$$reg),
9892 as_Register($src1$$reg),
9893 zr,
9894 Assembler::LSL, 0);
9895 %}
9896
9897 ins_pipe(ialu_reg);
9898 %}
9899
9900 // This pattern is automatically generated from aarch64_ad.m4
9901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9902 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9903 immI0 zero, iRegIorL2I src1, immI src2) %{
9904 match(Set dst (SubI zero (URShiftI src1 src2)));
9905
9906 ins_cost(1.9 * INSN_COST);
9907 format %{ "negw $dst, $src1, LSR $src2" %}
9908
9909 ins_encode %{
9910 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9911 Assembler::LSR, $src2$$constant & 0x1f);
9912 %}
9913
9914 ins_pipe(ialu_reg_shift);
9915 %}
9916
9917 // This pattern is automatically generated from aarch64_ad.m4
9918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9919 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9920 immI0 zero, iRegIorL2I src1, immI src2) %{
9921 match(Set dst (SubI zero (RShiftI src1 src2)));
9922
9923 ins_cost(1.9 * INSN_COST);
9924 format %{ "negw $dst, $src1, ASR $src2" %}
9925
9926 ins_encode %{
9927 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9928 Assembler::ASR, $src2$$constant & 0x1f);
9929 %}
9930
9931 ins_pipe(ialu_reg_shift);
9932 %}
9933
9934 // This pattern is automatically generated from aarch64_ad.m4
9935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9936 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9937 immI0 zero, iRegIorL2I src1, immI src2) %{
9938 match(Set dst (SubI zero (LShiftI src1 src2)));
9939
9940 ins_cost(1.9 * INSN_COST);
9941 format %{ "negw $dst, $src1, LSL $src2" %}
9942
9943 ins_encode %{
9944 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9945 Assembler::LSL, $src2$$constant & 0x1f);
9946 %}
9947
9948 ins_pipe(ialu_reg_shift);
9949 %}
9950
9951 // This pattern is automatically generated from aarch64_ad.m4
9952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9953 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9954 immL0 zero, iRegL src1, immI src2) %{
9955 match(Set dst (SubL zero (URShiftL src1 src2)));
9956
9957 ins_cost(1.9 * INSN_COST);
9958 format %{ "neg $dst, $src1, LSR $src2" %}
9959
9960 ins_encode %{
9961 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9962 Assembler::LSR, $src2$$constant & 0x3f);
9963 %}
9964
9965 ins_pipe(ialu_reg_shift);
9966 %}
9967
9968 // This pattern is automatically generated from aarch64_ad.m4
9969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9970 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9971 immL0 zero, iRegL src1, immI src2) %{
9972 match(Set dst (SubL zero (RShiftL src1 src2)));
9973
9974 ins_cost(1.9 * INSN_COST);
9975 format %{ "neg $dst, $src1, ASR $src2" %}
9976
9977 ins_encode %{
9978 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9979 Assembler::ASR, $src2$$constant & 0x3f);
9980 %}
9981
9982 ins_pipe(ialu_reg_shift);
9983 %}
9984
9985 // This pattern is automatically generated from aarch64_ad.m4
9986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9987 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9988 immL0 zero, iRegL src1, immI src2) %{
9989 match(Set dst (SubL zero (LShiftL src1 src2)));
9990
9991 ins_cost(1.9 * INSN_COST);
9992 format %{ "neg $dst, $src1, LSL $src2" %}
9993
9994 ins_encode %{
9995 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9996 Assembler::LSL, $src2$$constant & 0x3f);
9997 %}
9998
9999 ins_pipe(ialu_reg_shift);
10000 %}
10001
10002 // This pattern is automatically generated from aarch64_ad.m4
10003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10004 instruct AndI_reg_not_reg(iRegINoSp dst,
10005 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10006 match(Set dst (AndI src1 (XorI src2 m1)));
10007 ins_cost(INSN_COST);
10008 format %{ "bicw $dst, $src1, $src2" %}
10009
10010 ins_encode %{
10011 __ bicw(as_Register($dst$$reg),
10012 as_Register($src1$$reg),
10013 as_Register($src2$$reg),
10014 Assembler::LSL, 0);
10015 %}
10016
10017 ins_pipe(ialu_reg_reg);
10018 %}
10019
10020 // This pattern is automatically generated from aarch64_ad.m4
10021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10022 instruct AndL_reg_not_reg(iRegLNoSp dst,
10023 iRegL src1, iRegL src2, immL_M1 m1) %{
10024 match(Set dst (AndL src1 (XorL src2 m1)));
10025 ins_cost(INSN_COST);
10026 format %{ "bic $dst, $src1, $src2" %}
10027
10028 ins_encode %{
10029 __ bic(as_Register($dst$$reg),
10030 as_Register($src1$$reg),
10031 as_Register($src2$$reg),
10032 Assembler::LSL, 0);
10033 %}
10034
10035 ins_pipe(ialu_reg_reg);
10036 %}
10037
10038 // This pattern is automatically generated from aarch64_ad.m4
10039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10040 instruct OrI_reg_not_reg(iRegINoSp dst,
10041 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10042 match(Set dst (OrI src1 (XorI src2 m1)));
10043 ins_cost(INSN_COST);
10044 format %{ "ornw $dst, $src1, $src2" %}
10045
10046 ins_encode %{
10047 __ ornw(as_Register($dst$$reg),
10048 as_Register($src1$$reg),
10049 as_Register($src2$$reg),
10050 Assembler::LSL, 0);
10051 %}
10052
10053 ins_pipe(ialu_reg_reg);
10054 %}
10055
10056 // This pattern is automatically generated from aarch64_ad.m4
10057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10058 instruct OrL_reg_not_reg(iRegLNoSp dst,
10059 iRegL src1, iRegL src2, immL_M1 m1) %{
10060 match(Set dst (OrL src1 (XorL src2 m1)));
10061 ins_cost(INSN_COST);
10062 format %{ "orn $dst, $src1, $src2" %}
10063
10064 ins_encode %{
10065 __ orn(as_Register($dst$$reg),
10066 as_Register($src1$$reg),
10067 as_Register($src2$$reg),
10068 Assembler::LSL, 0);
10069 %}
10070
10071 ins_pipe(ialu_reg_reg);
10072 %}
10073
10074 // This pattern is automatically generated from aarch64_ad.m4
10075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10076 instruct XorI_reg_not_reg(iRegINoSp dst,
10077 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10078 match(Set dst (XorI m1 (XorI src2 src1)));
10079 ins_cost(INSN_COST);
10080 format %{ "eonw $dst, $src1, $src2" %}
10081
10082 ins_encode %{
10083 __ eonw(as_Register($dst$$reg),
10084 as_Register($src1$$reg),
10085 as_Register($src2$$reg),
10086 Assembler::LSL, 0);
10087 %}
10088
10089 ins_pipe(ialu_reg_reg);
10090 %}
10091
10092 // This pattern is automatically generated from aarch64_ad.m4
10093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10094 instruct XorL_reg_not_reg(iRegLNoSp dst,
10095 iRegL src1, iRegL src2, immL_M1 m1) %{
10096 match(Set dst (XorL m1 (XorL src2 src1)));
10097 ins_cost(INSN_COST);
10098 format %{ "eon $dst, $src1, $src2" %}
10099
10100 ins_encode %{
10101 __ eon(as_Register($dst$$reg),
10102 as_Register($src1$$reg),
10103 as_Register($src2$$reg),
10104 Assembler::LSL, 0);
10105 %}
10106
10107 ins_pipe(ialu_reg_reg);
10108 %}
10109
10110 // This pattern is automatically generated from aarch64_ad.m4
10111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10112 // val & (-1 ^ (val >>> shift)) ==> bicw
10113 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10114 iRegIorL2I src1, iRegIorL2I src2,
10115 immI src3, immI_M1 src4) %{
10116 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10117 ins_cost(1.9 * INSN_COST);
10118 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10119
10120 ins_encode %{
10121 __ bicw(as_Register($dst$$reg),
10122 as_Register($src1$$reg),
10123 as_Register($src2$$reg),
10124 Assembler::LSR,
10125 $src3$$constant & 0x1f);
10126 %}
10127
10128 ins_pipe(ialu_reg_reg_shift);
10129 %}
10130
10131 // This pattern is automatically generated from aarch64_ad.m4
10132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10133 // val & (-1 ^ (val >>> shift)) ==> bic
10134 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10135 iRegL src1, iRegL src2,
10136 immI src3, immL_M1 src4) %{
10137 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10138 ins_cost(1.9 * INSN_COST);
10139 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10140
10141 ins_encode %{
10142 __ bic(as_Register($dst$$reg),
10143 as_Register($src1$$reg),
10144 as_Register($src2$$reg),
10145 Assembler::LSR,
10146 $src3$$constant & 0x3f);
10147 %}
10148
10149 ins_pipe(ialu_reg_reg_shift);
10150 %}
10151
10152 // This pattern is automatically generated from aarch64_ad.m4
10153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10154 // val & (-1 ^ (val >> shift)) ==> bicw
10155 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10156 iRegIorL2I src1, iRegIorL2I src2,
10157 immI src3, immI_M1 src4) %{
10158 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10159 ins_cost(1.9 * INSN_COST);
10160 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10161
10162 ins_encode %{
10163 __ bicw(as_Register($dst$$reg),
10164 as_Register($src1$$reg),
10165 as_Register($src2$$reg),
10166 Assembler::ASR,
10167 $src3$$constant & 0x1f);
10168 %}
10169
10170 ins_pipe(ialu_reg_reg_shift);
10171 %}
10172
10173 // This pattern is automatically generated from aarch64_ad.m4
10174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10175 // val & (-1 ^ (val >> shift)) ==> bic
10176 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10177 iRegL src1, iRegL src2,
10178 immI src3, immL_M1 src4) %{
10179 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10180 ins_cost(1.9 * INSN_COST);
10181 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10182
10183 ins_encode %{
10184 __ bic(as_Register($dst$$reg),
10185 as_Register($src1$$reg),
10186 as_Register($src2$$reg),
10187 Assembler::ASR,
10188 $src3$$constant & 0x3f);
10189 %}
10190
10191 ins_pipe(ialu_reg_reg_shift);
10192 %}
10193
10194 // This pattern is automatically generated from aarch64_ad.m4
10195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10196 // val & (-1 ^ (val ror shift)) ==> bicw
10197 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10198 iRegIorL2I src1, iRegIorL2I src2,
10199 immI src3, immI_M1 src4) %{
10200 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10201 ins_cost(1.9 * INSN_COST);
10202 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10203
10204 ins_encode %{
10205 __ bicw(as_Register($dst$$reg),
10206 as_Register($src1$$reg),
10207 as_Register($src2$$reg),
10208 Assembler::ROR,
10209 $src3$$constant & 0x1f);
10210 %}
10211
10212 ins_pipe(ialu_reg_reg_shift);
10213 %}
10214
10215 // This pattern is automatically generated from aarch64_ad.m4
10216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10217 // val & (-1 ^ (val ror shift)) ==> bic
10218 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10219 iRegL src1, iRegL src2,
10220 immI src3, immL_M1 src4) %{
10221 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10222 ins_cost(1.9 * INSN_COST);
10223 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10224
10225 ins_encode %{
10226 __ bic(as_Register($dst$$reg),
10227 as_Register($src1$$reg),
10228 as_Register($src2$$reg),
10229 Assembler::ROR,
10230 $src3$$constant & 0x3f);
10231 %}
10232
10233 ins_pipe(ialu_reg_reg_shift);
10234 %}
10235
10236 // This pattern is automatically generated from aarch64_ad.m4
10237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10238 // val & (-1 ^ (val << shift)) ==> bicw
10239 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10240 iRegIorL2I src1, iRegIorL2I src2,
10241 immI src3, immI_M1 src4) %{
10242 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10243 ins_cost(1.9 * INSN_COST);
10244 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10245
10246 ins_encode %{
10247 __ bicw(as_Register($dst$$reg),
10248 as_Register($src1$$reg),
10249 as_Register($src2$$reg),
10250 Assembler::LSL,
10251 $src3$$constant & 0x1f);
10252 %}
10253
10254 ins_pipe(ialu_reg_reg_shift);
10255 %}
10256
10257 // This pattern is automatically generated from aarch64_ad.m4
10258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10259 // val & (-1 ^ (val << shift)) ==> bic
10260 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10261 iRegL src1, iRegL src2,
10262 immI src3, immL_M1 src4) %{
10263 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10264 ins_cost(1.9 * INSN_COST);
10265 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10266
10267 ins_encode %{
10268 __ bic(as_Register($dst$$reg),
10269 as_Register($src1$$reg),
10270 as_Register($src2$$reg),
10271 Assembler::LSL,
10272 $src3$$constant & 0x3f);
10273 %}
10274
10275 ins_pipe(ialu_reg_reg_shift);
10276 %}
10277
10278 // This pattern is automatically generated from aarch64_ad.m4
10279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10280 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10281 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10282 iRegIorL2I src1, iRegIorL2I src2,
10283 immI src3, immI_M1 src4) %{
10284 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10285 ins_cost(1.9 * INSN_COST);
10286 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10287
10288 ins_encode %{
10289 __ eonw(as_Register($dst$$reg),
10290 as_Register($src1$$reg),
10291 as_Register($src2$$reg),
10292 Assembler::LSR,
10293 $src3$$constant & 0x1f);
10294 %}
10295
10296 ins_pipe(ialu_reg_reg_shift);
10297 %}
10298
10299 // This pattern is automatically generated from aarch64_ad.m4
10300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10301 // val ^ (-1 ^ (val >>> shift)) ==> eon
10302 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10303 iRegL src1, iRegL src2,
10304 immI src3, immL_M1 src4) %{
10305 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10306 ins_cost(1.9 * INSN_COST);
10307 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10308
10309 ins_encode %{
10310 __ eon(as_Register($dst$$reg),
10311 as_Register($src1$$reg),
10312 as_Register($src2$$reg),
10313 Assembler::LSR,
10314 $src3$$constant & 0x3f);
10315 %}
10316
10317 ins_pipe(ialu_reg_reg_shift);
10318 %}
10319
10320 // This pattern is automatically generated from aarch64_ad.m4
10321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10322 // val ^ (-1 ^ (val >> shift)) ==> eonw
10323 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10324 iRegIorL2I src1, iRegIorL2I src2,
10325 immI src3, immI_M1 src4) %{
10326 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10327 ins_cost(1.9 * INSN_COST);
10328 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10329
10330 ins_encode %{
10331 __ eonw(as_Register($dst$$reg),
10332 as_Register($src1$$reg),
10333 as_Register($src2$$reg),
10334 Assembler::ASR,
10335 $src3$$constant & 0x1f);
10336 %}
10337
10338 ins_pipe(ialu_reg_reg_shift);
10339 %}
10340
10341 // This pattern is automatically generated from aarch64_ad.m4
10342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10343 // val ^ (-1 ^ (val >> shift)) ==> eon
10344 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10345 iRegL src1, iRegL src2,
10346 immI src3, immL_M1 src4) %{
10347 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10348 ins_cost(1.9 * INSN_COST);
10349 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10350
10351 ins_encode %{
10352 __ eon(as_Register($dst$$reg),
10353 as_Register($src1$$reg),
10354 as_Register($src2$$reg),
10355 Assembler::ASR,
10356 $src3$$constant & 0x3f);
10357 %}
10358
10359 ins_pipe(ialu_reg_reg_shift);
10360 %}
10361
10362 // This pattern is automatically generated from aarch64_ad.m4
10363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10364 // val ^ (-1 ^ (val ror shift)) ==> eonw
10365 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10366 iRegIorL2I src1, iRegIorL2I src2,
10367 immI src3, immI_M1 src4) %{
10368 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10369 ins_cost(1.9 * INSN_COST);
10370 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10371
10372 ins_encode %{
10373 __ eonw(as_Register($dst$$reg),
10374 as_Register($src1$$reg),
10375 as_Register($src2$$reg),
10376 Assembler::ROR,
10377 $src3$$constant & 0x1f);
10378 %}
10379
10380 ins_pipe(ialu_reg_reg_shift);
10381 %}
10382
10383 // This pattern is automatically generated from aarch64_ad.m4
10384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10385 // val ^ (-1 ^ (val ror shift)) ==> eon
10386 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10387 iRegL src1, iRegL src2,
10388 immI src3, immL_M1 src4) %{
10389 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10390 ins_cost(1.9 * INSN_COST);
10391 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10392
10393 ins_encode %{
10394 __ eon(as_Register($dst$$reg),
10395 as_Register($src1$$reg),
10396 as_Register($src2$$reg),
10397 Assembler::ROR,
10398 $src3$$constant & 0x3f);
10399 %}
10400
10401 ins_pipe(ialu_reg_reg_shift);
10402 %}
10403
10404 // This pattern is automatically generated from aarch64_ad.m4
10405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10406 // val ^ (-1 ^ (val << shift)) ==> eonw
10407 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10408 iRegIorL2I src1, iRegIorL2I src2,
10409 immI src3, immI_M1 src4) %{
10410 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10411 ins_cost(1.9 * INSN_COST);
10412 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10413
10414 ins_encode %{
10415 __ eonw(as_Register($dst$$reg),
10416 as_Register($src1$$reg),
10417 as_Register($src2$$reg),
10418 Assembler::LSL,
10419 $src3$$constant & 0x1f);
10420 %}
10421
10422 ins_pipe(ialu_reg_reg_shift);
10423 %}
10424
10425 // This pattern is automatically generated from aarch64_ad.m4
10426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10427 // val ^ (-1 ^ (val << shift)) ==> eon
10428 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10429 iRegL src1, iRegL src2,
10430 immI src3, immL_M1 src4) %{
10431 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10432 ins_cost(1.9 * INSN_COST);
10433 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10434
10435 ins_encode %{
10436 __ eon(as_Register($dst$$reg),
10437 as_Register($src1$$reg),
10438 as_Register($src2$$reg),
10439 Assembler::LSL,
10440 $src3$$constant & 0x3f);
10441 %}
10442
10443 ins_pipe(ialu_reg_reg_shift);
10444 %}
10445
10446 // This pattern is automatically generated from aarch64_ad.m4
10447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10448 // val | (-1 ^ (val >>> shift)) ==> ornw
10449 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10450 iRegIorL2I src1, iRegIorL2I src2,
10451 immI src3, immI_M1 src4) %{
10452 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10453 ins_cost(1.9 * INSN_COST);
10454 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10455
10456 ins_encode %{
10457 __ ornw(as_Register($dst$$reg),
10458 as_Register($src1$$reg),
10459 as_Register($src2$$reg),
10460 Assembler::LSR,
10461 $src3$$constant & 0x1f);
10462 %}
10463
10464 ins_pipe(ialu_reg_reg_shift);
10465 %}
10466
10467 // This pattern is automatically generated from aarch64_ad.m4
10468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10469 // val | (-1 ^ (val >>> shift)) ==> orn
10470 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10471 iRegL src1, iRegL src2,
10472 immI src3, immL_M1 src4) %{
10473 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10474 ins_cost(1.9 * INSN_COST);
10475 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10476
10477 ins_encode %{
10478 __ orn(as_Register($dst$$reg),
10479 as_Register($src1$$reg),
10480 as_Register($src2$$reg),
10481 Assembler::LSR,
10482 $src3$$constant & 0x3f);
10483 %}
10484
10485 ins_pipe(ialu_reg_reg_shift);
10486 %}
10487
10488 // This pattern is automatically generated from aarch64_ad.m4
10489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10490 // val | (-1 ^ (val >> shift)) ==> ornw
10491 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10492 iRegIorL2I src1, iRegIorL2I src2,
10493 immI src3, immI_M1 src4) %{
10494 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10495 ins_cost(1.9 * INSN_COST);
10496 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10497
10498 ins_encode %{
10499 __ ornw(as_Register($dst$$reg),
10500 as_Register($src1$$reg),
10501 as_Register($src2$$reg),
10502 Assembler::ASR,
10503 $src3$$constant & 0x1f);
10504 %}
10505
10506 ins_pipe(ialu_reg_reg_shift);
10507 %}
10508
10509 // This pattern is automatically generated from aarch64_ad.m4
10510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10511 // val | (-1 ^ (val >> shift)) ==> orn
10512 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10513 iRegL src1, iRegL src2,
10514 immI src3, immL_M1 src4) %{
10515 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10516 ins_cost(1.9 * INSN_COST);
10517 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10518
10519 ins_encode %{
10520 __ orn(as_Register($dst$$reg),
10521 as_Register($src1$$reg),
10522 as_Register($src2$$reg),
10523 Assembler::ASR,
10524 $src3$$constant & 0x3f);
10525 %}
10526
10527 ins_pipe(ialu_reg_reg_shift);
10528 %}
10529
10530 // This pattern is automatically generated from aarch64_ad.m4
10531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10532 // val | (-1 ^ (val ror shift)) ==> ornw
10533 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10534 iRegIorL2I src1, iRegIorL2I src2,
10535 immI src3, immI_M1 src4) %{
10536 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10537 ins_cost(1.9 * INSN_COST);
10538 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10539
10540 ins_encode %{
10541 __ ornw(as_Register($dst$$reg),
10542 as_Register($src1$$reg),
10543 as_Register($src2$$reg),
10544 Assembler::ROR,
10545 $src3$$constant & 0x1f);
10546 %}
10547
10548 ins_pipe(ialu_reg_reg_shift);
10549 %}
10550
10551 // This pattern is automatically generated from aarch64_ad.m4
10552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10553 // val | (-1 ^ (val ror shift)) ==> orn
10554 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10555 iRegL src1, iRegL src2,
10556 immI src3, immL_M1 src4) %{
10557 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10558 ins_cost(1.9 * INSN_COST);
10559 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10560
10561 ins_encode %{
10562 __ orn(as_Register($dst$$reg),
10563 as_Register($src1$$reg),
10564 as_Register($src2$$reg),
10565 Assembler::ROR,
10566 $src3$$constant & 0x3f);
10567 %}
10568
10569 ins_pipe(ialu_reg_reg_shift);
10570 %}
10571
10572 // This pattern is automatically generated from aarch64_ad.m4
10573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10574 // val | (-1 ^ (val << shift)) ==> ornw
10575 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10576 iRegIorL2I src1, iRegIorL2I src2,
10577 immI src3, immI_M1 src4) %{
10578 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10579 ins_cost(1.9 * INSN_COST);
10580 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10581
10582 ins_encode %{
10583 __ ornw(as_Register($dst$$reg),
10584 as_Register($src1$$reg),
10585 as_Register($src2$$reg),
10586 Assembler::LSL,
10587 $src3$$constant & 0x1f);
10588 %}
10589
10590 ins_pipe(ialu_reg_reg_shift);
10591 %}
10592
10593 // This pattern is automatically generated from aarch64_ad.m4
10594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10595 // val | (-1 ^ (val << shift)) ==> orn
10596 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10597 iRegL src1, iRegL src2,
10598 immI src3, immL_M1 src4) %{
10599 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10600 ins_cost(1.9 * INSN_COST);
10601 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10602
10603 ins_encode %{
10604 __ orn(as_Register($dst$$reg),
10605 as_Register($src1$$reg),
10606 as_Register($src2$$reg),
10607 Assembler::LSL,
10608 $src3$$constant & 0x3f);
10609 %}
10610
10611 ins_pipe(ialu_reg_reg_shift);
10612 %}
10613
10614 // This pattern is automatically generated from aarch64_ad.m4
10615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10616 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10617 iRegIorL2I src1, iRegIorL2I src2,
10618 immI src3) %{
10619 match(Set dst (AndI src1 (URShiftI src2 src3)));
10620
10621 ins_cost(1.9 * INSN_COST);
10622 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10623
10624 ins_encode %{
10625 __ andw(as_Register($dst$$reg),
10626 as_Register($src1$$reg),
10627 as_Register($src2$$reg),
10628 Assembler::LSR,
10629 $src3$$constant & 0x1f);
10630 %}
10631
10632 ins_pipe(ialu_reg_reg_shift);
10633 %}
10634
10635 // This pattern is automatically generated from aarch64_ad.m4
10636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10637 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10638 iRegL src1, iRegL src2,
10639 immI src3) %{
10640 match(Set dst (AndL src1 (URShiftL src2 src3)));
10641
10642 ins_cost(1.9 * INSN_COST);
10643 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10644
10645 ins_encode %{
10646 __ andr(as_Register($dst$$reg),
10647 as_Register($src1$$reg),
10648 as_Register($src2$$reg),
10649 Assembler::LSR,
10650 $src3$$constant & 0x3f);
10651 %}
10652
10653 ins_pipe(ialu_reg_reg_shift);
10654 %}
10655
10656 // This pattern is automatically generated from aarch64_ad.m4
10657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10658 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10659 iRegIorL2I src1, iRegIorL2I src2,
10660 immI src3) %{
10661 match(Set dst (AndI src1 (RShiftI src2 src3)));
10662
10663 ins_cost(1.9 * INSN_COST);
10664 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10665
10666 ins_encode %{
10667 __ andw(as_Register($dst$$reg),
10668 as_Register($src1$$reg),
10669 as_Register($src2$$reg),
10670 Assembler::ASR,
10671 $src3$$constant & 0x1f);
10672 %}
10673
10674 ins_pipe(ialu_reg_reg_shift);
10675 %}
10676
10677 // This pattern is automatically generated from aarch64_ad.m4
10678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10679 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10680 iRegL src1, iRegL src2,
10681 immI src3) %{
10682 match(Set dst (AndL src1 (RShiftL src2 src3)));
10683
10684 ins_cost(1.9 * INSN_COST);
10685 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10686
10687 ins_encode %{
10688 __ andr(as_Register($dst$$reg),
10689 as_Register($src1$$reg),
10690 as_Register($src2$$reg),
10691 Assembler::ASR,
10692 $src3$$constant & 0x3f);
10693 %}
10694
10695 ins_pipe(ialu_reg_reg_shift);
10696 %}
10697
10698 // This pattern is automatically generated from aarch64_ad.m4
10699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10700 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10701 iRegIorL2I src1, iRegIorL2I src2,
10702 immI src3) %{
10703 match(Set dst (AndI src1 (LShiftI src2 src3)));
10704
10705 ins_cost(1.9 * INSN_COST);
10706 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10707
10708 ins_encode %{
10709 __ andw(as_Register($dst$$reg),
10710 as_Register($src1$$reg),
10711 as_Register($src2$$reg),
10712 Assembler::LSL,
10713 $src3$$constant & 0x1f);
10714 %}
10715
10716 ins_pipe(ialu_reg_reg_shift);
10717 %}
10718
10719 // This pattern is automatically generated from aarch64_ad.m4
10720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10721 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10722 iRegL src1, iRegL src2,
10723 immI src3) %{
10724 match(Set dst (AndL src1 (LShiftL src2 src3)));
10725
10726 ins_cost(1.9 * INSN_COST);
10727 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10728
10729 ins_encode %{
10730 __ andr(as_Register($dst$$reg),
10731 as_Register($src1$$reg),
10732 as_Register($src2$$reg),
10733 Assembler::LSL,
10734 $src3$$constant & 0x3f);
10735 %}
10736
10737 ins_pipe(ialu_reg_reg_shift);
10738 %}
10739
10740 // This pattern is automatically generated from aarch64_ad.m4
10741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10742 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10743 iRegIorL2I src1, iRegIorL2I src2,
10744 immI src3) %{
10745 match(Set dst (AndI src1 (RotateRight src2 src3)));
10746
10747 ins_cost(1.9 * INSN_COST);
10748 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10749
10750 ins_encode %{
10751 __ andw(as_Register($dst$$reg),
10752 as_Register($src1$$reg),
10753 as_Register($src2$$reg),
10754 Assembler::ROR,
10755 $src3$$constant & 0x1f);
10756 %}
10757
10758 ins_pipe(ialu_reg_reg_shift);
10759 %}
10760
10761 // This pattern is automatically generated from aarch64_ad.m4
10762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10763 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10764 iRegL src1, iRegL src2,
10765 immI src3) %{
10766 match(Set dst (AndL src1 (RotateRight src2 src3)));
10767
10768 ins_cost(1.9 * INSN_COST);
10769 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10770
10771 ins_encode %{
10772 __ andr(as_Register($dst$$reg),
10773 as_Register($src1$$reg),
10774 as_Register($src2$$reg),
10775 Assembler::ROR,
10776 $src3$$constant & 0x3f);
10777 %}
10778
10779 ins_pipe(ialu_reg_reg_shift);
10780 %}
10781
10782 // This pattern is automatically generated from aarch64_ad.m4
10783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10784 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10785 iRegIorL2I src1, iRegIorL2I src2,
10786 immI src3) %{
10787 match(Set dst (XorI src1 (URShiftI src2 src3)));
10788
10789 ins_cost(1.9 * INSN_COST);
10790 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10791
10792 ins_encode %{
10793 __ eorw(as_Register($dst$$reg),
10794 as_Register($src1$$reg),
10795 as_Register($src2$$reg),
10796 Assembler::LSR,
10797 $src3$$constant & 0x1f);
10798 %}
10799
10800 ins_pipe(ialu_reg_reg_shift);
10801 %}
10802
10803 // This pattern is automatically generated from aarch64_ad.m4
10804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10805 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10806 iRegL src1, iRegL src2,
10807 immI src3) %{
10808 match(Set dst (XorL src1 (URShiftL src2 src3)));
10809
10810 ins_cost(1.9 * INSN_COST);
10811 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10812
10813 ins_encode %{
10814 __ eor(as_Register($dst$$reg),
10815 as_Register($src1$$reg),
10816 as_Register($src2$$reg),
10817 Assembler::LSR,
10818 $src3$$constant & 0x3f);
10819 %}
10820
10821 ins_pipe(ialu_reg_reg_shift);
10822 %}
10823
10824 // This pattern is automatically generated from aarch64_ad.m4
10825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10826 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10827 iRegIorL2I src1, iRegIorL2I src2,
10828 immI src3) %{
10829 match(Set dst (XorI src1 (RShiftI src2 src3)));
10830
10831 ins_cost(1.9 * INSN_COST);
10832 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10833
10834 ins_encode %{
10835 __ eorw(as_Register($dst$$reg),
10836 as_Register($src1$$reg),
10837 as_Register($src2$$reg),
10838 Assembler::ASR,
10839 $src3$$constant & 0x1f);
10840 %}
10841
10842 ins_pipe(ialu_reg_reg_shift);
10843 %}
10844
10845 // This pattern is automatically generated from aarch64_ad.m4
10846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10847 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10848 iRegL src1, iRegL src2,
10849 immI src3) %{
10850 match(Set dst (XorL src1 (RShiftL src2 src3)));
10851
10852 ins_cost(1.9 * INSN_COST);
10853 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10854
10855 ins_encode %{
10856 __ eor(as_Register($dst$$reg),
10857 as_Register($src1$$reg),
10858 as_Register($src2$$reg),
10859 Assembler::ASR,
10860 $src3$$constant & 0x3f);
10861 %}
10862
10863 ins_pipe(ialu_reg_reg_shift);
10864 %}
10865
10866 // This pattern is automatically generated from aarch64_ad.m4
10867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10868 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10869 iRegIorL2I src1, iRegIorL2I src2,
10870 immI src3) %{
10871 match(Set dst (XorI src1 (LShiftI src2 src3)));
10872
10873 ins_cost(1.9 * INSN_COST);
10874 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10875
10876 ins_encode %{
10877 __ eorw(as_Register($dst$$reg),
10878 as_Register($src1$$reg),
10879 as_Register($src2$$reg),
10880 Assembler::LSL,
10881 $src3$$constant & 0x1f);
10882 %}
10883
10884 ins_pipe(ialu_reg_reg_shift);
10885 %}
10886
10887 // This pattern is automatically generated from aarch64_ad.m4
10888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10889 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10890 iRegL src1, iRegL src2,
10891 immI src3) %{
10892 match(Set dst (XorL src1 (LShiftL src2 src3)));
10893
10894 ins_cost(1.9 * INSN_COST);
10895 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10896
10897 ins_encode %{
10898 __ eor(as_Register($dst$$reg),
10899 as_Register($src1$$reg),
10900 as_Register($src2$$reg),
10901 Assembler::LSL,
10902 $src3$$constant & 0x3f);
10903 %}
10904
10905 ins_pipe(ialu_reg_reg_shift);
10906 %}
10907
10908 // This pattern is automatically generated from aarch64_ad.m4
10909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10910 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10911 iRegIorL2I src1, iRegIorL2I src2,
10912 immI src3) %{
10913 match(Set dst (XorI src1 (RotateRight src2 src3)));
10914
10915 ins_cost(1.9 * INSN_COST);
10916 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10917
10918 ins_encode %{
10919 __ eorw(as_Register($dst$$reg),
10920 as_Register($src1$$reg),
10921 as_Register($src2$$reg),
10922 Assembler::ROR,
10923 $src3$$constant & 0x1f);
10924 %}
10925
10926 ins_pipe(ialu_reg_reg_shift);
10927 %}
10928
10929 // This pattern is automatically generated from aarch64_ad.m4
10930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10931 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10932 iRegL src1, iRegL src2,
10933 immI src3) %{
10934 match(Set dst (XorL src1 (RotateRight src2 src3)));
10935
10936 ins_cost(1.9 * INSN_COST);
10937 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10938
10939 ins_encode %{
10940 __ eor(as_Register($dst$$reg),
10941 as_Register($src1$$reg),
10942 as_Register($src2$$reg),
10943 Assembler::ROR,
10944 $src3$$constant & 0x3f);
10945 %}
10946
10947 ins_pipe(ialu_reg_reg_shift);
10948 %}
10949
10950 // This pattern is automatically generated from aarch64_ad.m4
10951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10952 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10953 iRegIorL2I src1, iRegIorL2I src2,
10954 immI src3) %{
10955 match(Set dst (OrI src1 (URShiftI src2 src3)));
10956
10957 ins_cost(1.9 * INSN_COST);
10958 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10959
10960 ins_encode %{
10961 __ orrw(as_Register($dst$$reg),
10962 as_Register($src1$$reg),
10963 as_Register($src2$$reg),
10964 Assembler::LSR,
10965 $src3$$constant & 0x1f);
10966 %}
10967
10968 ins_pipe(ialu_reg_reg_shift);
10969 %}
10970
10971 // This pattern is automatically generated from aarch64_ad.m4
10972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10973 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10974 iRegL src1, iRegL src2,
10975 immI src3) %{
10976 match(Set dst (OrL src1 (URShiftL src2 src3)));
10977
10978 ins_cost(1.9 * INSN_COST);
10979 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10980
10981 ins_encode %{
10982 __ orr(as_Register($dst$$reg),
10983 as_Register($src1$$reg),
10984 as_Register($src2$$reg),
10985 Assembler::LSR,
10986 $src3$$constant & 0x3f);
10987 %}
10988
10989 ins_pipe(ialu_reg_reg_shift);
10990 %}
10991
10992 // This pattern is automatically generated from aarch64_ad.m4
10993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10994 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10995 iRegIorL2I src1, iRegIorL2I src2,
10996 immI src3) %{
10997 match(Set dst (OrI src1 (RShiftI src2 src3)));
10998
10999 ins_cost(1.9 * INSN_COST);
11000 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11001
11002 ins_encode %{
11003 __ orrw(as_Register($dst$$reg),
11004 as_Register($src1$$reg),
11005 as_Register($src2$$reg),
11006 Assembler::ASR,
11007 $src3$$constant & 0x1f);
11008 %}
11009
11010 ins_pipe(ialu_reg_reg_shift);
11011 %}
11012
11013 // This pattern is automatically generated from aarch64_ad.m4
11014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11015 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11016 iRegL src1, iRegL src2,
11017 immI src3) %{
11018 match(Set dst (OrL src1 (RShiftL src2 src3)));
11019
11020 ins_cost(1.9 * INSN_COST);
11021 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11022
11023 ins_encode %{
11024 __ orr(as_Register($dst$$reg),
11025 as_Register($src1$$reg),
11026 as_Register($src2$$reg),
11027 Assembler::ASR,
11028 $src3$$constant & 0x3f);
11029 %}
11030
11031 ins_pipe(ialu_reg_reg_shift);
11032 %}
11033
11034 // This pattern is automatically generated from aarch64_ad.m4
11035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11036 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11037 iRegIorL2I src1, iRegIorL2I src2,
11038 immI src3) %{
11039 match(Set dst (OrI src1 (LShiftI src2 src3)));
11040
11041 ins_cost(1.9 * INSN_COST);
11042 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11043
11044 ins_encode %{
11045 __ orrw(as_Register($dst$$reg),
11046 as_Register($src1$$reg),
11047 as_Register($src2$$reg),
11048 Assembler::LSL,
11049 $src3$$constant & 0x1f);
11050 %}
11051
11052 ins_pipe(ialu_reg_reg_shift);
11053 %}
11054
11055 // This pattern is automatically generated from aarch64_ad.m4
11056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11057 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11058 iRegL src1, iRegL src2,
11059 immI src3) %{
11060 match(Set dst (OrL src1 (LShiftL src2 src3)));
11061
11062 ins_cost(1.9 * INSN_COST);
11063 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11064
11065 ins_encode %{
11066 __ orr(as_Register($dst$$reg),
11067 as_Register($src1$$reg),
11068 as_Register($src2$$reg),
11069 Assembler::LSL,
11070 $src3$$constant & 0x3f);
11071 %}
11072
11073 ins_pipe(ialu_reg_reg_shift);
11074 %}
11075
11076 // This pattern is automatically generated from aarch64_ad.m4
11077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11078 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11079 iRegIorL2I src1, iRegIorL2I src2,
11080 immI src3) %{
11081 match(Set dst (OrI src1 (RotateRight src2 src3)));
11082
11083 ins_cost(1.9 * INSN_COST);
11084 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11085
11086 ins_encode %{
11087 __ orrw(as_Register($dst$$reg),
11088 as_Register($src1$$reg),
11089 as_Register($src2$$reg),
11090 Assembler::ROR,
11091 $src3$$constant & 0x1f);
11092 %}
11093
11094 ins_pipe(ialu_reg_reg_shift);
11095 %}
11096
11097 // This pattern is automatically generated from aarch64_ad.m4
11098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11099 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11100 iRegL src1, iRegL src2,
11101 immI src3) %{
11102 match(Set dst (OrL src1 (RotateRight src2 src3)));
11103
11104 ins_cost(1.9 * INSN_COST);
11105 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11106
11107 ins_encode %{
11108 __ orr(as_Register($dst$$reg),
11109 as_Register($src1$$reg),
11110 as_Register($src2$$reg),
11111 Assembler::ROR,
11112 $src3$$constant & 0x3f);
11113 %}
11114
11115 ins_pipe(ialu_reg_reg_shift);
11116 %}
11117
11118 // This pattern is automatically generated from aarch64_ad.m4
11119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11120 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11121 iRegIorL2I src1, iRegIorL2I src2,
11122 immI src3) %{
11123 match(Set dst (AddI src1 (URShiftI src2 src3)));
11124
11125 ins_cost(1.9 * INSN_COST);
11126 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11127
11128 ins_encode %{
11129 __ addw(as_Register($dst$$reg),
11130 as_Register($src1$$reg),
11131 as_Register($src2$$reg),
11132 Assembler::LSR,
11133 $src3$$constant & 0x1f);
11134 %}
11135
11136 ins_pipe(ialu_reg_reg_shift);
11137 %}
11138
11139 // This pattern is automatically generated from aarch64_ad.m4
11140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11141 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11142 iRegL src1, iRegL src2,
11143 immI src3) %{
11144 match(Set dst (AddL src1 (URShiftL src2 src3)));
11145
11146 ins_cost(1.9 * INSN_COST);
11147 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11148
11149 ins_encode %{
11150 __ add(as_Register($dst$$reg),
11151 as_Register($src1$$reg),
11152 as_Register($src2$$reg),
11153 Assembler::LSR,
11154 $src3$$constant & 0x3f);
11155 %}
11156
11157 ins_pipe(ialu_reg_reg_shift);
11158 %}
11159
11160 // This pattern is automatically generated from aarch64_ad.m4
11161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11162 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11163 iRegIorL2I src1, iRegIorL2I src2,
11164 immI src3) %{
11165 match(Set dst (AddI src1 (RShiftI src2 src3)));
11166
11167 ins_cost(1.9 * INSN_COST);
11168 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11169
11170 ins_encode %{
11171 __ addw(as_Register($dst$$reg),
11172 as_Register($src1$$reg),
11173 as_Register($src2$$reg),
11174 Assembler::ASR,
11175 $src3$$constant & 0x1f);
11176 %}
11177
11178 ins_pipe(ialu_reg_reg_shift);
11179 %}
11180
11181 // This pattern is automatically generated from aarch64_ad.m4
11182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11183 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11184 iRegL src1, iRegL src2,
11185 immI src3) %{
11186 match(Set dst (AddL src1 (RShiftL src2 src3)));
11187
11188 ins_cost(1.9 * INSN_COST);
11189 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11190
11191 ins_encode %{
11192 __ add(as_Register($dst$$reg),
11193 as_Register($src1$$reg),
11194 as_Register($src2$$reg),
11195 Assembler::ASR,
11196 $src3$$constant & 0x3f);
11197 %}
11198
11199 ins_pipe(ialu_reg_reg_shift);
11200 %}
11201
11202 // This pattern is automatically generated from aarch64_ad.m4
11203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11204 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11205 iRegIorL2I src1, iRegIorL2I src2,
11206 immI src3) %{
11207 match(Set dst (AddI src1 (LShiftI src2 src3)));
11208
11209 ins_cost(1.9 * INSN_COST);
11210 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11211
11212 ins_encode %{
11213 __ addw(as_Register($dst$$reg),
11214 as_Register($src1$$reg),
11215 as_Register($src2$$reg),
11216 Assembler::LSL,
11217 $src3$$constant & 0x1f);
11218 %}
11219
11220 ins_pipe(ialu_reg_reg_shift);
11221 %}
11222
11223 // This pattern is automatically generated from aarch64_ad.m4
11224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11225 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11226 iRegL src1, iRegL src2,
11227 immI src3) %{
11228 match(Set dst (AddL src1 (LShiftL src2 src3)));
11229
11230 ins_cost(1.9 * INSN_COST);
11231 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11232
11233 ins_encode %{
11234 __ add(as_Register($dst$$reg),
11235 as_Register($src1$$reg),
11236 as_Register($src2$$reg),
11237 Assembler::LSL,
11238 $src3$$constant & 0x3f);
11239 %}
11240
11241 ins_pipe(ialu_reg_reg_shift);
11242 %}
11243
11244 // This pattern is automatically generated from aarch64_ad.m4
11245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11246 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11247 iRegIorL2I src1, iRegIorL2I src2,
11248 immI src3) %{
11249 match(Set dst (SubI src1 (URShiftI src2 src3)));
11250
11251 ins_cost(1.9 * INSN_COST);
11252 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11253
11254 ins_encode %{
11255 __ subw(as_Register($dst$$reg),
11256 as_Register($src1$$reg),
11257 as_Register($src2$$reg),
11258 Assembler::LSR,
11259 $src3$$constant & 0x1f);
11260 %}
11261
11262 ins_pipe(ialu_reg_reg_shift);
11263 %}
11264
11265 // This pattern is automatically generated from aarch64_ad.m4
11266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11267 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11268 iRegL src1, iRegL src2,
11269 immI src3) %{
11270 match(Set dst (SubL src1 (URShiftL src2 src3)));
11271
11272 ins_cost(1.9 * INSN_COST);
11273 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11274
11275 ins_encode %{
11276 __ sub(as_Register($dst$$reg),
11277 as_Register($src1$$reg),
11278 as_Register($src2$$reg),
11279 Assembler::LSR,
11280 $src3$$constant & 0x3f);
11281 %}
11282
11283 ins_pipe(ialu_reg_reg_shift);
11284 %}
11285
11286 // This pattern is automatically generated from aarch64_ad.m4
11287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11288 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11289 iRegIorL2I src1, iRegIorL2I src2,
11290 immI src3) %{
11291 match(Set dst (SubI src1 (RShiftI src2 src3)));
11292
11293 ins_cost(1.9 * INSN_COST);
11294 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11295
11296 ins_encode %{
11297 __ subw(as_Register($dst$$reg),
11298 as_Register($src1$$reg),
11299 as_Register($src2$$reg),
11300 Assembler::ASR,
11301 $src3$$constant & 0x1f);
11302 %}
11303
11304 ins_pipe(ialu_reg_reg_shift);
11305 %}
11306
11307 // This pattern is automatically generated from aarch64_ad.m4
11308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11309 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11310 iRegL src1, iRegL src2,
11311 immI src3) %{
11312 match(Set dst (SubL src1 (RShiftL src2 src3)));
11313
11314 ins_cost(1.9 * INSN_COST);
11315 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11316
11317 ins_encode %{
11318 __ sub(as_Register($dst$$reg),
11319 as_Register($src1$$reg),
11320 as_Register($src2$$reg),
11321 Assembler::ASR,
11322 $src3$$constant & 0x3f);
11323 %}
11324
11325 ins_pipe(ialu_reg_reg_shift);
11326 %}
11327
11328 // This pattern is automatically generated from aarch64_ad.m4
11329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11330 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11331 iRegIorL2I src1, iRegIorL2I src2,
11332 immI src3) %{
11333 match(Set dst (SubI src1 (LShiftI src2 src3)));
11334
11335 ins_cost(1.9 * INSN_COST);
11336 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11337
11338 ins_encode %{
11339 __ subw(as_Register($dst$$reg),
11340 as_Register($src1$$reg),
11341 as_Register($src2$$reg),
11342 Assembler::LSL,
11343 $src3$$constant & 0x1f);
11344 %}
11345
11346 ins_pipe(ialu_reg_reg_shift);
11347 %}
11348
11349 // This pattern is automatically generated from aarch64_ad.m4
11350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11351 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11352 iRegL src1, iRegL src2,
11353 immI src3) %{
11354 match(Set dst (SubL src1 (LShiftL src2 src3)));
11355
11356 ins_cost(1.9 * INSN_COST);
11357 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11358
11359 ins_encode %{
11360 __ sub(as_Register($dst$$reg),
11361 as_Register($src1$$reg),
11362 as_Register($src2$$reg),
11363 Assembler::LSL,
11364 $src3$$constant & 0x3f);
11365 %}
11366
11367 ins_pipe(ialu_reg_reg_shift);
11368 %}
11369
11370 // This pattern is automatically generated from aarch64_ad.m4
11371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11372
11373 // Shift Left followed by Shift Right.
11374 // This idiom is used by the compiler for the i2b bytecode etc.
11375 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11376 %{
11377 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11378 ins_cost(INSN_COST * 2);
11379 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11380 ins_encode %{
11381 int lshift = $lshift_count$$constant & 63;
11382 int rshift = $rshift_count$$constant & 63;
11383 int s = 63 - lshift;
11384 int r = (rshift - lshift) & 63;
11385 __ sbfm(as_Register($dst$$reg),
11386 as_Register($src$$reg),
11387 r, s);
11388 %}
11389
11390 ins_pipe(ialu_reg_shift);
11391 %}
11392
11393 // This pattern is automatically generated from aarch64_ad.m4
11394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11395
11396 // Shift Left followed by Shift Right.
11397 // This idiom is used by the compiler for the i2b bytecode etc.
11398 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11399 %{
11400 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11401 ins_cost(INSN_COST * 2);
11402 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11403 ins_encode %{
11404 int lshift = $lshift_count$$constant & 31;
11405 int rshift = $rshift_count$$constant & 31;
11406 int s = 31 - lshift;
11407 int r = (rshift - lshift) & 31;
11408 __ sbfmw(as_Register($dst$$reg),
11409 as_Register($src$$reg),
11410 r, s);
11411 %}
11412
11413 ins_pipe(ialu_reg_shift);
11414 %}
11415
11416 // This pattern is automatically generated from aarch64_ad.m4
11417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11418
11419 // Shift Left followed by Shift Right.
11420 // This idiom is used by the compiler for the i2b bytecode etc.
11421 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11422 %{
11423 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11424 ins_cost(INSN_COST * 2);
11425 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11426 ins_encode %{
11427 int lshift = $lshift_count$$constant & 63;
11428 int rshift = $rshift_count$$constant & 63;
11429 int s = 63 - lshift;
11430 int r = (rshift - lshift) & 63;
11431 __ ubfm(as_Register($dst$$reg),
11432 as_Register($src$$reg),
11433 r, s);
11434 %}
11435
11436 ins_pipe(ialu_reg_shift);
11437 %}
11438
11439 // This pattern is automatically generated from aarch64_ad.m4
11440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11441
11442 // Shift Left followed by Shift Right.
11443 // This idiom is used by the compiler for the i2b bytecode etc.
11444 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11445 %{
11446 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11447 ins_cost(INSN_COST * 2);
11448 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11449 ins_encode %{
11450 int lshift = $lshift_count$$constant & 31;
11451 int rshift = $rshift_count$$constant & 31;
11452 int s = 31 - lshift;
11453 int r = (rshift - lshift) & 31;
11454 __ ubfmw(as_Register($dst$$reg),
11455 as_Register($src$$reg),
11456 r, s);
11457 %}
11458
11459 ins_pipe(ialu_reg_shift);
11460 %}
11461
11462 // Bitfield extract with shift & mask
11463
11464 // This pattern is automatically generated from aarch64_ad.m4
11465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11466 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11467 %{
11468 match(Set dst (AndI (URShiftI src rshift) mask));
11469 // Make sure we are not going to exceed what ubfxw can do.
11470 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11471
11472 ins_cost(INSN_COST);
11473 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11474 ins_encode %{
11475 int rshift = $rshift$$constant & 31;
11476 intptr_t mask = $mask$$constant;
11477 int width = exact_log2(mask+1);
11478 __ ubfxw(as_Register($dst$$reg),
11479 as_Register($src$$reg), rshift, width);
11480 %}
11481 ins_pipe(ialu_reg_shift);
11482 %}
11483
11484 // This pattern is automatically generated from aarch64_ad.m4
11485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11486 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11487 %{
11488 match(Set dst (AndL (URShiftL src rshift) mask));
11489 // Make sure we are not going to exceed what ubfx can do.
11490 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11491
11492 ins_cost(INSN_COST);
11493 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11494 ins_encode %{
11495 int rshift = $rshift$$constant & 63;
11496 intptr_t mask = $mask$$constant;
11497 int width = exact_log2_long(mask+1);
11498 __ ubfx(as_Register($dst$$reg),
11499 as_Register($src$$reg), rshift, width);
11500 %}
11501 ins_pipe(ialu_reg_shift);
11502 %}
11503
11504
11505 // This pattern is automatically generated from aarch64_ad.m4
11506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11507
11508 // We can use ubfx when extending an And with a mask when we know mask
11509 // is positive. We know that because immI_bitmask guarantees it.
11510 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11511 %{
11512 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11513 // Make sure we are not going to exceed what ubfxw can do.
11514 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11515
11516 ins_cost(INSN_COST * 2);
11517 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11518 ins_encode %{
11519 int rshift = $rshift$$constant & 31;
11520 intptr_t mask = $mask$$constant;
11521 int width = exact_log2(mask+1);
11522 __ ubfx(as_Register($dst$$reg),
11523 as_Register($src$$reg), rshift, width);
11524 %}
11525 ins_pipe(ialu_reg_shift);
11526 %}
11527
11528
11529 // This pattern is automatically generated from aarch64_ad.m4
11530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11531
11532 // We can use ubfiz when masking by a positive number and then left shifting the result.
11533 // We know that the mask is positive because immI_bitmask guarantees it.
11534 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11535 %{
11536 match(Set dst (LShiftI (AndI src mask) lshift));
11537 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11538
11539 ins_cost(INSN_COST);
11540 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11541 ins_encode %{
11542 int lshift = $lshift$$constant & 31;
11543 intptr_t mask = $mask$$constant;
11544 int width = exact_log2(mask+1);
11545 __ ubfizw(as_Register($dst$$reg),
11546 as_Register($src$$reg), lshift, width);
11547 %}
11548 ins_pipe(ialu_reg_shift);
11549 %}
11550
11551 // This pattern is automatically generated from aarch64_ad.m4
11552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11553
11554 // We can use ubfiz when masking by a positive number and then left shifting the result.
11555 // We know that the mask is positive because immL_bitmask guarantees it.
11556 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11557 %{
11558 match(Set dst (LShiftL (AndL src mask) lshift));
11559 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11560
11561 ins_cost(INSN_COST);
11562 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11563 ins_encode %{
11564 int lshift = $lshift$$constant & 63;
11565 intptr_t mask = $mask$$constant;
11566 int width = exact_log2_long(mask+1);
11567 __ ubfiz(as_Register($dst$$reg),
11568 as_Register($src$$reg), lshift, width);
11569 %}
11570 ins_pipe(ialu_reg_shift);
11571 %}
11572
11573 // This pattern is automatically generated from aarch64_ad.m4
11574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11575
11576 // We can use ubfiz when masking by a positive number and then left shifting the result.
11577 // We know that the mask is positive because immI_bitmask guarantees it.
11578 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11579 %{
11580 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11581 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11582
11583 ins_cost(INSN_COST);
11584 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11585 ins_encode %{
11586 int lshift = $lshift$$constant & 31;
11587 intptr_t mask = $mask$$constant;
11588 int width = exact_log2(mask+1);
11589 __ ubfizw(as_Register($dst$$reg),
11590 as_Register($src$$reg), lshift, width);
11591 %}
11592 ins_pipe(ialu_reg_shift);
11593 %}
11594
11595 // This pattern is automatically generated from aarch64_ad.m4
11596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11597
11598 // We can use ubfiz when masking by a positive number and then left shifting the result.
11599 // We know that the mask is positive because immL_bitmask guarantees it.
11600 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11601 %{
11602 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11603 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11604
11605 ins_cost(INSN_COST);
11606 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11607 ins_encode %{
11608 int lshift = $lshift$$constant & 63;
11609 intptr_t mask = $mask$$constant;
11610 int width = exact_log2_long(mask+1);
11611 __ ubfiz(as_Register($dst$$reg),
11612 as_Register($src$$reg), lshift, width);
11613 %}
11614 ins_pipe(ialu_reg_shift);
11615 %}
11616
11617
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620
11621 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11622 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11623 %{
11624 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11625 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11626
11627 ins_cost(INSN_COST);
11628 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11629 ins_encode %{
11630 int lshift = $lshift$$constant & 63;
11631 intptr_t mask = $mask$$constant;
11632 int width = exact_log2(mask+1);
11633 __ ubfiz(as_Register($dst$$reg),
11634 as_Register($src$$reg), lshift, width);
11635 %}
11636 ins_pipe(ialu_reg_shift);
11637 %}
11638
11639 // This pattern is automatically generated from aarch64_ad.m4
11640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11641
11642 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11643 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11644 %{
11645 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11646 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11647
11648 ins_cost(INSN_COST);
11649 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11650 ins_encode %{
11651 int lshift = $lshift$$constant & 31;
11652 intptr_t mask = $mask$$constant;
11653 int width = exact_log2(mask+1);
11654 __ ubfiz(as_Register($dst$$reg),
11655 as_Register($src$$reg), lshift, width);
11656 %}
11657 ins_pipe(ialu_reg_shift);
11658 %}
11659
11660 // This pattern is automatically generated from aarch64_ad.m4
11661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11662
11663 // Can skip int2long conversions after AND with small bitmask
11664 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11665 %{
11666 match(Set dst (ConvI2L (AndI src msk)));
11667 ins_cost(INSN_COST);
11668 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11669 ins_encode %{
11670 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11671 %}
11672 ins_pipe(ialu_reg_shift);
11673 %}
11674
11675
11676 // Rotations
11677
11678 // This pattern is automatically generated from aarch64_ad.m4
11679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11680 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11681 %{
11682 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11683 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11684
11685 ins_cost(INSN_COST);
11686 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11687
11688 ins_encode %{
11689 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11690 $rshift$$constant & 63);
11691 %}
11692 ins_pipe(ialu_reg_reg_extr);
11693 %}
11694
11695
11696 // This pattern is automatically generated from aarch64_ad.m4
11697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11698 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11699 %{
11700 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11701 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11702
11703 ins_cost(INSN_COST);
11704 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11705
11706 ins_encode %{
11707 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11708 $rshift$$constant & 31);
11709 %}
11710 ins_pipe(ialu_reg_reg_extr);
11711 %}
11712
11713
11714 // This pattern is automatically generated from aarch64_ad.m4
11715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11716 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11717 %{
11718 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11719 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11720
11721 ins_cost(INSN_COST);
11722 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11723
11724 ins_encode %{
11725 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11726 $rshift$$constant & 63);
11727 %}
11728 ins_pipe(ialu_reg_reg_extr);
11729 %}
11730
11731
11732 // This pattern is automatically generated from aarch64_ad.m4
11733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11734 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11735 %{
11736 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11737 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11738
11739 ins_cost(INSN_COST);
11740 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11741
11742 ins_encode %{
11743 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11744 $rshift$$constant & 31);
11745 %}
11746 ins_pipe(ialu_reg_reg_extr);
11747 %}
11748
11749 // This pattern is automatically generated from aarch64_ad.m4
11750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11751 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11752 %{
11753 match(Set dst (RotateRight src shift));
11754
11755 ins_cost(INSN_COST);
11756 format %{ "ror $dst, $src, $shift" %}
11757
11758 ins_encode %{
11759 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11760 $shift$$constant & 0x1f);
11761 %}
11762 ins_pipe(ialu_reg_reg_vshift);
11763 %}
11764
11765 // This pattern is automatically generated from aarch64_ad.m4
11766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11767 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11768 %{
11769 match(Set dst (RotateRight src shift));
11770
11771 ins_cost(INSN_COST);
11772 format %{ "ror $dst, $src, $shift" %}
11773
11774 ins_encode %{
11775 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11776 $shift$$constant & 0x3f);
11777 %}
11778 ins_pipe(ialu_reg_reg_vshift);
11779 %}
11780
11781 // This pattern is automatically generated from aarch64_ad.m4
11782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11783 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11784 %{
11785 match(Set dst (RotateRight src shift));
11786
11787 ins_cost(INSN_COST);
11788 format %{ "ror $dst, $src, $shift" %}
11789
11790 ins_encode %{
11791 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11792 %}
11793 ins_pipe(ialu_reg_reg_vshift);
11794 %}
11795
11796 // This pattern is automatically generated from aarch64_ad.m4
11797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11798 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11799 %{
11800 match(Set dst (RotateRight src shift));
11801
11802 ins_cost(INSN_COST);
11803 format %{ "ror $dst, $src, $shift" %}
11804
11805 ins_encode %{
11806 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11807 %}
11808 ins_pipe(ialu_reg_reg_vshift);
11809 %}
11810
11811 // This pattern is automatically generated from aarch64_ad.m4
11812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11813 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11814 %{
11815 match(Set dst (RotateLeft src shift));
11816
11817 ins_cost(INSN_COST);
11818 format %{ "rol $dst, $src, $shift" %}
11819
11820 ins_encode %{
11821 __ subw(rscratch1, zr, as_Register($shift$$reg));
11822 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11823 %}
11824 ins_pipe(ialu_reg_reg_vshift);
11825 %}
11826
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11830 %{
11831 match(Set dst (RotateLeft src shift));
11832
11833 ins_cost(INSN_COST);
11834 format %{ "rol $dst, $src, $shift" %}
11835
11836 ins_encode %{
11837 __ subw(rscratch1, zr, as_Register($shift$$reg));
11838 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11839 %}
11840 ins_pipe(ialu_reg_reg_vshift);
11841 %}
11842
11843
11844 // Add/subtract (extended)
11845
11846 // This pattern is automatically generated from aarch64_ad.m4
11847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11848 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11849 %{
11850 match(Set dst (AddL src1 (ConvI2L src2)));
11851 ins_cost(INSN_COST);
11852 format %{ "add $dst, $src1, $src2, sxtw" %}
11853
11854 ins_encode %{
11855 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11856 as_Register($src2$$reg), ext::sxtw);
11857 %}
11858 ins_pipe(ialu_reg_reg);
11859 %}
11860
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11864 %{
11865 match(Set dst (SubL src1 (ConvI2L src2)));
11866 ins_cost(INSN_COST);
11867 format %{ "sub $dst, $src1, $src2, sxtw" %}
11868
11869 ins_encode %{
11870 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11871 as_Register($src2$$reg), ext::sxtw);
11872 %}
11873 ins_pipe(ialu_reg_reg);
11874 %}
11875
11876 // This pattern is automatically generated from aarch64_ad.m4
11877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11878 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11879 %{
11880 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11881 ins_cost(INSN_COST);
11882 format %{ "add $dst, $src1, $src2, sxth" %}
11883
11884 ins_encode %{
11885 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11886 as_Register($src2$$reg), ext::sxth);
11887 %}
11888 ins_pipe(ialu_reg_reg);
11889 %}
11890
11891 // This pattern is automatically generated from aarch64_ad.m4
11892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11893 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11894 %{
11895 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11896 ins_cost(INSN_COST);
11897 format %{ "add $dst, $src1, $src2, sxtb" %}
11898
11899 ins_encode %{
11900 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11901 as_Register($src2$$reg), ext::sxtb);
11902 %}
11903 ins_pipe(ialu_reg_reg);
11904 %}
11905
11906 // This pattern is automatically generated from aarch64_ad.m4
11907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11908 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11909 %{
11910 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11911 ins_cost(INSN_COST);
11912 format %{ "add $dst, $src1, $src2, uxtb" %}
11913
11914 ins_encode %{
11915 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11916 as_Register($src2$$reg), ext::uxtb);
11917 %}
11918 ins_pipe(ialu_reg_reg);
11919 %}
11920
11921 // This pattern is automatically generated from aarch64_ad.m4
11922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11923 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11924 %{
11925 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11926 ins_cost(INSN_COST);
11927 format %{ "add $dst, $src1, $src2, sxth" %}
11928
11929 ins_encode %{
11930 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11931 as_Register($src2$$reg), ext::sxth);
11932 %}
11933 ins_pipe(ialu_reg_reg);
11934 %}
11935
11936 // This pattern is automatically generated from aarch64_ad.m4
11937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11938 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11939 %{
11940 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11941 ins_cost(INSN_COST);
11942 format %{ "add $dst, $src1, $src2, sxtw" %}
11943
11944 ins_encode %{
11945 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11946 as_Register($src2$$reg), ext::sxtw);
11947 %}
11948 ins_pipe(ialu_reg_reg);
11949 %}
11950
11951 // This pattern is automatically generated from aarch64_ad.m4
11952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11953 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11954 %{
11955 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11956 ins_cost(INSN_COST);
11957 format %{ "add $dst, $src1, $src2, sxtb" %}
11958
11959 ins_encode %{
11960 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11961 as_Register($src2$$reg), ext::sxtb);
11962 %}
11963 ins_pipe(ialu_reg_reg);
11964 %}
11965
11966 // This pattern is automatically generated from aarch64_ad.m4
11967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11968 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11969 %{
11970 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11971 ins_cost(INSN_COST);
11972 format %{ "add $dst, $src1, $src2, uxtb" %}
11973
11974 ins_encode %{
11975 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11976 as_Register($src2$$reg), ext::uxtb);
11977 %}
11978 ins_pipe(ialu_reg_reg);
11979 %}
11980
11981 // This pattern is automatically generated from aarch64_ad.m4
11982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11983 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11984 %{
11985 match(Set dst (AddI src1 (AndI src2 mask)));
11986 ins_cost(INSN_COST);
11987 format %{ "addw $dst, $src1, $src2, uxtb" %}
11988
11989 ins_encode %{
11990 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11991 as_Register($src2$$reg), ext::uxtb);
11992 %}
11993 ins_pipe(ialu_reg_reg);
11994 %}
11995
11996 // This pattern is automatically generated from aarch64_ad.m4
11997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11998 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11999 %{
12000 match(Set dst (AddI src1 (AndI src2 mask)));
12001 ins_cost(INSN_COST);
12002 format %{ "addw $dst, $src1, $src2, uxth" %}
12003
12004 ins_encode %{
12005 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12006 as_Register($src2$$reg), ext::uxth);
12007 %}
12008 ins_pipe(ialu_reg_reg);
12009 %}
12010
12011 // This pattern is automatically generated from aarch64_ad.m4
12012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12013 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12014 %{
12015 match(Set dst (AddL src1 (AndL src2 mask)));
12016 ins_cost(INSN_COST);
12017 format %{ "add $dst, $src1, $src2, uxtb" %}
12018
12019 ins_encode %{
12020 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12021 as_Register($src2$$reg), ext::uxtb);
12022 %}
12023 ins_pipe(ialu_reg_reg);
12024 %}
12025
12026 // This pattern is automatically generated from aarch64_ad.m4
12027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12028 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12029 %{
12030 match(Set dst (AddL src1 (AndL src2 mask)));
12031 ins_cost(INSN_COST);
12032 format %{ "add $dst, $src1, $src2, uxth" %}
12033
12034 ins_encode %{
12035 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12036 as_Register($src2$$reg), ext::uxth);
12037 %}
12038 ins_pipe(ialu_reg_reg);
12039 %}
12040
12041 // This pattern is automatically generated from aarch64_ad.m4
12042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12043 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12044 %{
12045 match(Set dst (AddL src1 (AndL src2 mask)));
12046 ins_cost(INSN_COST);
12047 format %{ "add $dst, $src1, $src2, uxtw" %}
12048
12049 ins_encode %{
12050 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12051 as_Register($src2$$reg), ext::uxtw);
12052 %}
12053 ins_pipe(ialu_reg_reg);
12054 %}
12055
12056 // This pattern is automatically generated from aarch64_ad.m4
12057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12058 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12059 %{
12060 match(Set dst (SubI src1 (AndI src2 mask)));
12061 ins_cost(INSN_COST);
12062 format %{ "subw $dst, $src1, $src2, uxtb" %}
12063
12064 ins_encode %{
12065 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12066 as_Register($src2$$reg), ext::uxtb);
12067 %}
12068 ins_pipe(ialu_reg_reg);
12069 %}
12070
12071 // This pattern is automatically generated from aarch64_ad.m4
12072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12073 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12074 %{
12075 match(Set dst (SubI src1 (AndI src2 mask)));
12076 ins_cost(INSN_COST);
12077 format %{ "subw $dst, $src1, $src2, uxth" %}
12078
12079 ins_encode %{
12080 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12081 as_Register($src2$$reg), ext::uxth);
12082 %}
12083 ins_pipe(ialu_reg_reg);
12084 %}
12085
12086 // This pattern is automatically generated from aarch64_ad.m4
12087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12088 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12089 %{
12090 match(Set dst (SubL src1 (AndL src2 mask)));
12091 ins_cost(INSN_COST);
12092 format %{ "sub $dst, $src1, $src2, uxtb" %}
12093
12094 ins_encode %{
12095 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12096 as_Register($src2$$reg), ext::uxtb);
12097 %}
12098 ins_pipe(ialu_reg_reg);
12099 %}
12100
12101 // This pattern is automatically generated from aarch64_ad.m4
12102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12103 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12104 %{
12105 match(Set dst (SubL src1 (AndL src2 mask)));
12106 ins_cost(INSN_COST);
12107 format %{ "sub $dst, $src1, $src2, uxth" %}
12108
12109 ins_encode %{
12110 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12111 as_Register($src2$$reg), ext::uxth);
12112 %}
12113 ins_pipe(ialu_reg_reg);
12114 %}
12115
12116 // This pattern is automatically generated from aarch64_ad.m4
12117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12118 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12119 %{
12120 match(Set dst (SubL src1 (AndL src2 mask)));
12121 ins_cost(INSN_COST);
12122 format %{ "sub $dst, $src1, $src2, uxtw" %}
12123
12124 ins_encode %{
12125 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12126 as_Register($src2$$reg), ext::uxtw);
12127 %}
12128 ins_pipe(ialu_reg_reg);
12129 %}
12130
12131
12132 // This pattern is automatically generated from aarch64_ad.m4
12133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12134 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12135 %{
12136 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12137 ins_cost(1.9 * INSN_COST);
12138 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12139
12140 ins_encode %{
12141 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12142 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12143 %}
12144 ins_pipe(ialu_reg_reg_shift);
12145 %}
12146
12147 // This pattern is automatically generated from aarch64_ad.m4
12148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12149 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12150 %{
12151 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12152 ins_cost(1.9 * INSN_COST);
12153 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12154
12155 ins_encode %{
12156 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12157 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12158 %}
12159 ins_pipe(ialu_reg_reg_shift);
12160 %}
12161
12162 // This pattern is automatically generated from aarch64_ad.m4
12163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12164 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12165 %{
12166 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12167 ins_cost(1.9 * INSN_COST);
12168 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12169
12170 ins_encode %{
12171 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12172 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12173 %}
12174 ins_pipe(ialu_reg_reg_shift);
12175 %}
12176
12177 // This pattern is automatically generated from aarch64_ad.m4
12178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12179 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12180 %{
12181 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12182 ins_cost(1.9 * INSN_COST);
12183 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12184
12185 ins_encode %{
12186 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12187 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12188 %}
12189 ins_pipe(ialu_reg_reg_shift);
12190 %}
12191
12192 // This pattern is automatically generated from aarch64_ad.m4
12193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12194 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12195 %{
12196 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12197 ins_cost(1.9 * INSN_COST);
12198 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12199
12200 ins_encode %{
12201 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12202 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12203 %}
12204 ins_pipe(ialu_reg_reg_shift);
12205 %}
12206
12207 // This pattern is automatically generated from aarch64_ad.m4
12208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12209 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12210 %{
12211 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12212 ins_cost(1.9 * INSN_COST);
12213 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12214
12215 ins_encode %{
12216 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12217 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12218 %}
12219 ins_pipe(ialu_reg_reg_shift);
12220 %}
12221
12222 // This pattern is automatically generated from aarch64_ad.m4
12223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12224 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12225 %{
12226 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12227 ins_cost(1.9 * INSN_COST);
12228 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12229
12230 ins_encode %{
12231 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12232 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12233 %}
12234 ins_pipe(ialu_reg_reg_shift);
12235 %}
12236
12237 // This pattern is automatically generated from aarch64_ad.m4
12238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12239 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12240 %{
12241 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12242 ins_cost(1.9 * INSN_COST);
12243 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12244
12245 ins_encode %{
12246 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12247 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12248 %}
12249 ins_pipe(ialu_reg_reg_shift);
12250 %}
12251
12252 // This pattern is automatically generated from aarch64_ad.m4
12253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12254 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12255 %{
12256 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12257 ins_cost(1.9 * INSN_COST);
12258 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12259
12260 ins_encode %{
12261 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12262 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12263 %}
12264 ins_pipe(ialu_reg_reg_shift);
12265 %}
12266
12267 // This pattern is automatically generated from aarch64_ad.m4
12268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12269 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12270 %{
12271 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12272 ins_cost(1.9 * INSN_COST);
12273 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12274
12275 ins_encode %{
12276 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12277 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12278 %}
12279 ins_pipe(ialu_reg_reg_shift);
12280 %}
12281
12282 // This pattern is automatically generated from aarch64_ad.m4
12283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12284 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12285 %{
12286 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12287 ins_cost(1.9 * INSN_COST);
12288 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12289
12290 ins_encode %{
12291 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12292 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12293 %}
12294 ins_pipe(ialu_reg_reg_shift);
12295 %}
12296
12297 // This pattern is automatically generated from aarch64_ad.m4
12298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12299 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12300 %{
12301 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12302 ins_cost(1.9 * INSN_COST);
12303 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12304
12305 ins_encode %{
12306 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12307 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12308 %}
12309 ins_pipe(ialu_reg_reg_shift);
12310 %}
12311
12312 // This pattern is automatically generated from aarch64_ad.m4
12313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12314 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12315 %{
12316 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12317 ins_cost(1.9 * INSN_COST);
12318 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12319
12320 ins_encode %{
12321 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12322 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12323 %}
12324 ins_pipe(ialu_reg_reg_shift);
12325 %}
12326
12327 // This pattern is automatically generated from aarch64_ad.m4
12328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12329 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12330 %{
12331 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12332 ins_cost(1.9 * INSN_COST);
12333 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12334
12335 ins_encode %{
12336 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12337 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12338 %}
12339 ins_pipe(ialu_reg_reg_shift);
12340 %}
12341
12342 // This pattern is automatically generated from aarch64_ad.m4
12343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12344 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12345 %{
12346 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12347 ins_cost(1.9 * INSN_COST);
12348 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12349
12350 ins_encode %{
12351 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12352 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12353 %}
12354 ins_pipe(ialu_reg_reg_shift);
12355 %}
12356
12357 // This pattern is automatically generated from aarch64_ad.m4
12358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12359 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12360 %{
12361 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12362 ins_cost(1.9 * INSN_COST);
12363 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12364
12365 ins_encode %{
12366 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12367 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12368 %}
12369 ins_pipe(ialu_reg_reg_shift);
12370 %}
12371
12372 // This pattern is automatically generated from aarch64_ad.m4
12373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12374 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12375 %{
12376 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12377 ins_cost(1.9 * INSN_COST);
12378 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12379
12380 ins_encode %{
12381 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12382 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12383 %}
12384 ins_pipe(ialu_reg_reg_shift);
12385 %}
12386
12387 // This pattern is automatically generated from aarch64_ad.m4
12388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12389 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12390 %{
12391 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12392 ins_cost(1.9 * INSN_COST);
12393 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12394
12395 ins_encode %{
12396 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12397 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12398 %}
12399 ins_pipe(ialu_reg_reg_shift);
12400 %}
12401
12402 // This pattern is automatically generated from aarch64_ad.m4
12403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12404 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12405 %{
12406 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12407 ins_cost(1.9 * INSN_COST);
12408 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12409
12410 ins_encode %{
12411 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12412 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12413 %}
12414 ins_pipe(ialu_reg_reg_shift);
12415 %}
12416
12417 // This pattern is automatically generated from aarch64_ad.m4
12418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12419 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12420 %{
12421 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12422 ins_cost(1.9 * INSN_COST);
12423 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12424
12425 ins_encode %{
12426 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12427 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12428 %}
12429 ins_pipe(ialu_reg_reg_shift);
12430 %}
12431
12432 // This pattern is automatically generated from aarch64_ad.m4
12433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12434 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12435 %{
12436 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12437 ins_cost(1.9 * INSN_COST);
12438 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12439
12440 ins_encode %{
12441 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12442 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12443 %}
12444 ins_pipe(ialu_reg_reg_shift);
12445 %}
12446
12447 // This pattern is automatically generated from aarch64_ad.m4
12448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12449 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12450 %{
12451 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12452 ins_cost(1.9 * INSN_COST);
12453 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12454
12455 ins_encode %{
12456 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12457 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12458 %}
12459 ins_pipe(ialu_reg_reg_shift);
12460 %}
12461
12462 // This pattern is automatically generated from aarch64_ad.m4
12463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12464 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12465 %{
12466 effect(DEF dst, USE src1, USE src2, USE cr);
12467 ins_cost(INSN_COST * 2);
12468 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12469
12470 ins_encode %{
12471 __ cselw($dst$$Register,
12472 $src1$$Register,
12473 $src2$$Register,
12474 Assembler::LT);
12475 %}
12476 ins_pipe(icond_reg_reg);
12477 %}
12478
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12482 %{
12483 effect(DEF dst, USE src1, USE src2, USE cr);
12484 ins_cost(INSN_COST * 2);
12485 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12486
12487 ins_encode %{
12488 __ cselw($dst$$Register,
12489 $src1$$Register,
12490 $src2$$Register,
12491 Assembler::GT);
12492 %}
12493 ins_pipe(icond_reg_reg);
12494 %}
12495
12496 // This pattern is automatically generated from aarch64_ad.m4
12497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12498 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12499 %{
12500 effect(DEF dst, USE src1, USE cr);
12501 ins_cost(INSN_COST * 2);
12502 format %{ "cselw $dst, $src1, zr lt\t" %}
12503
12504 ins_encode %{
12505 __ cselw($dst$$Register,
12506 $src1$$Register,
12507 zr,
12508 Assembler::LT);
12509 %}
12510 ins_pipe(icond_reg);
12511 %}
12512
12513 // This pattern is automatically generated from aarch64_ad.m4
12514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12515 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12516 %{
12517 effect(DEF dst, USE src1, USE cr);
12518 ins_cost(INSN_COST * 2);
12519 format %{ "cselw $dst, $src1, zr gt\t" %}
12520
12521 ins_encode %{
12522 __ cselw($dst$$Register,
12523 $src1$$Register,
12524 zr,
12525 Assembler::GT);
12526 %}
12527 ins_pipe(icond_reg);
12528 %}
12529
12530 // This pattern is automatically generated from aarch64_ad.m4
12531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12532 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12533 %{
12534 effect(DEF dst, USE src1, USE cr);
12535 ins_cost(INSN_COST * 2);
12536 format %{ "csincw $dst, $src1, zr le\t" %}
12537
12538 ins_encode %{
12539 __ csincw($dst$$Register,
12540 $src1$$Register,
12541 zr,
12542 Assembler::LE);
12543 %}
12544 ins_pipe(icond_reg);
12545 %}
12546
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12550 %{
12551 effect(DEF dst, USE src1, USE cr);
12552 ins_cost(INSN_COST * 2);
12553 format %{ "csincw $dst, $src1, zr gt\t" %}
12554
12555 ins_encode %{
12556 __ csincw($dst$$Register,
12557 $src1$$Register,
12558 zr,
12559 Assembler::GT);
12560 %}
12561 ins_pipe(icond_reg);
12562 %}
12563
12564 // This pattern is automatically generated from aarch64_ad.m4
12565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12566 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12567 %{
12568 effect(DEF dst, USE src1, USE cr);
12569 ins_cost(INSN_COST * 2);
12570 format %{ "csinvw $dst, $src1, zr lt\t" %}
12571
12572 ins_encode %{
12573 __ csinvw($dst$$Register,
12574 $src1$$Register,
12575 zr,
12576 Assembler::LT);
12577 %}
12578 ins_pipe(icond_reg);
12579 %}
12580
12581 // This pattern is automatically generated from aarch64_ad.m4
12582 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12583 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12584 %{
12585 effect(DEF dst, USE src1, USE cr);
12586 ins_cost(INSN_COST * 2);
12587 format %{ "csinvw $dst, $src1, zr ge\t" %}
12588
12589 ins_encode %{
12590 __ csinvw($dst$$Register,
12591 $src1$$Register,
12592 zr,
12593 Assembler::GE);
12594 %}
12595 ins_pipe(icond_reg);
12596 %}
12597
12598 // This pattern is automatically generated from aarch64_ad.m4
12599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12600 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12601 %{
12602 match(Set dst (MinI src imm));
12603 ins_cost(INSN_COST * 3);
12604 expand %{
12605 rFlagsReg cr;
12606 compI_reg_imm0(cr, src);
12607 cmovI_reg_imm0_lt(dst, src, cr);
12608 %}
12609 %}
12610
12611 // This pattern is automatically generated from aarch64_ad.m4
12612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12613 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12614 %{
12615 match(Set dst (MinI imm src));
12616 ins_cost(INSN_COST * 3);
12617 expand %{
12618 rFlagsReg cr;
12619 compI_reg_imm0(cr, src);
12620 cmovI_reg_imm0_lt(dst, src, cr);
12621 %}
12622 %}
12623
12624 // This pattern is automatically generated from aarch64_ad.m4
12625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12626 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12627 %{
12628 match(Set dst (MinI src imm));
12629 ins_cost(INSN_COST * 3);
12630 expand %{
12631 rFlagsReg cr;
12632 compI_reg_imm0(cr, src);
12633 cmovI_reg_imm1_le(dst, src, cr);
12634 %}
12635 %}
12636
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12640 %{
12641 match(Set dst (MinI imm src));
12642 ins_cost(INSN_COST * 3);
12643 expand %{
12644 rFlagsReg cr;
12645 compI_reg_imm0(cr, src);
12646 cmovI_reg_imm1_le(dst, src, cr);
12647 %}
12648 %}
12649
12650 // This pattern is automatically generated from aarch64_ad.m4
12651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12652 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12653 %{
12654 match(Set dst (MinI src imm));
12655 ins_cost(INSN_COST * 3);
12656 expand %{
12657 rFlagsReg cr;
12658 compI_reg_imm0(cr, src);
12659 cmovI_reg_immM1_lt(dst, src, cr);
12660 %}
12661 %}
12662
12663 // This pattern is automatically generated from aarch64_ad.m4
12664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12665 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12666 %{
12667 match(Set dst (MinI imm src));
12668 ins_cost(INSN_COST * 3);
12669 expand %{
12670 rFlagsReg cr;
12671 compI_reg_imm0(cr, src);
12672 cmovI_reg_immM1_lt(dst, src, cr);
12673 %}
12674 %}
12675
12676 // This pattern is automatically generated from aarch64_ad.m4
12677 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12678 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12679 %{
12680 match(Set dst (MaxI src imm));
12681 ins_cost(INSN_COST * 3);
12682 expand %{
12683 rFlagsReg cr;
12684 compI_reg_imm0(cr, src);
12685 cmovI_reg_imm0_gt(dst, src, cr);
12686 %}
12687 %}
12688
12689 // This pattern is automatically generated from aarch64_ad.m4
12690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12691 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12692 %{
12693 match(Set dst (MaxI imm src));
12694 ins_cost(INSN_COST * 3);
12695 expand %{
12696 rFlagsReg cr;
12697 compI_reg_imm0(cr, src);
12698 cmovI_reg_imm0_gt(dst, src, cr);
12699 %}
12700 %}
12701
12702 // This pattern is automatically generated from aarch64_ad.m4
12703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12704 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12705 %{
12706 match(Set dst (MaxI src imm));
12707 ins_cost(INSN_COST * 3);
12708 expand %{
12709 rFlagsReg cr;
12710 compI_reg_imm0(cr, src);
12711 cmovI_reg_imm1_gt(dst, src, cr);
12712 %}
12713 %}
12714
12715 // This pattern is automatically generated from aarch64_ad.m4
12716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12717 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12718 %{
12719 match(Set dst (MaxI imm src));
12720 ins_cost(INSN_COST * 3);
12721 expand %{
12722 rFlagsReg cr;
12723 compI_reg_imm0(cr, src);
12724 cmovI_reg_imm1_gt(dst, src, cr);
12725 %}
12726 %}
12727
12728 // This pattern is automatically generated from aarch64_ad.m4
12729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12730 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12731 %{
12732 match(Set dst (MaxI src imm));
12733 ins_cost(INSN_COST * 3);
12734 expand %{
12735 rFlagsReg cr;
12736 compI_reg_imm0(cr, src);
12737 cmovI_reg_immM1_ge(dst, src, cr);
12738 %}
12739 %}
12740
12741 // This pattern is automatically generated from aarch64_ad.m4
12742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12743 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12744 %{
12745 match(Set dst (MaxI imm src));
12746 ins_cost(INSN_COST * 3);
12747 expand %{
12748 rFlagsReg cr;
12749 compI_reg_imm0(cr, src);
12750 cmovI_reg_immM1_ge(dst, src, cr);
12751 %}
12752 %}
12753
12754 // This pattern is automatically generated from aarch64_ad.m4
12755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12756 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12757 %{
12758 match(Set dst (ReverseI src));
12759 ins_cost(INSN_COST);
12760 format %{ "rbitw $dst, $src" %}
12761 ins_encode %{
12762 __ rbitw($dst$$Register, $src$$Register);
12763 %}
12764 ins_pipe(ialu_reg);
12765 %}
12766
12767 // This pattern is automatically generated from aarch64_ad.m4
12768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12769 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12770 %{
12771 match(Set dst (ReverseL src));
12772 ins_cost(INSN_COST);
12773 format %{ "rbit $dst, $src" %}
12774 ins_encode %{
12775 __ rbit($dst$$Register, $src$$Register);
12776 %}
12777 ins_pipe(ialu_reg);
12778 %}
12779
12780
12781 // END This section of the file is automatically generated. Do not edit --------------
12782
12783
12784 // ============================================================================
12785 // Floating Point Arithmetic Instructions
12786
12787 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12788 match(Set dst (AddHF src1 src2));
12789 format %{ "faddh $dst, $src1, $src2" %}
12790 ins_encode %{
12791 __ faddh($dst$$FloatRegister,
12792 $src1$$FloatRegister,
12793 $src2$$FloatRegister);
12794 %}
12795 ins_pipe(fp_dop_reg_reg_s);
12796 %}
12797
12798 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12799 match(Set dst (AddF src1 src2));
12800
12801 ins_cost(INSN_COST * 5);
12802 format %{ "fadds $dst, $src1, $src2" %}
12803
12804 ins_encode %{
12805 __ fadds(as_FloatRegister($dst$$reg),
12806 as_FloatRegister($src1$$reg),
12807 as_FloatRegister($src2$$reg));
12808 %}
12809
12810 ins_pipe(fp_dop_reg_reg_s);
12811 %}
12812
12813 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12814 match(Set dst (AddD src1 src2));
12815
12816 ins_cost(INSN_COST * 5);
12817 format %{ "faddd $dst, $src1, $src2" %}
12818
12819 ins_encode %{
12820 __ faddd(as_FloatRegister($dst$$reg),
12821 as_FloatRegister($src1$$reg),
12822 as_FloatRegister($src2$$reg));
12823 %}
12824
12825 ins_pipe(fp_dop_reg_reg_d);
12826 %}
12827
12828 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12829 match(Set dst (SubHF src1 src2));
12830 format %{ "fsubh $dst, $src1, $src2" %}
12831 ins_encode %{
12832 __ fsubh($dst$$FloatRegister,
12833 $src1$$FloatRegister,
12834 $src2$$FloatRegister);
12835 %}
12836 ins_pipe(fp_dop_reg_reg_s);
12837 %}
12838
12839 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12840 match(Set dst (SubF src1 src2));
12841
12842 ins_cost(INSN_COST * 5);
12843 format %{ "fsubs $dst, $src1, $src2" %}
12844
12845 ins_encode %{
12846 __ fsubs(as_FloatRegister($dst$$reg),
12847 as_FloatRegister($src1$$reg),
12848 as_FloatRegister($src2$$reg));
12849 %}
12850
12851 ins_pipe(fp_dop_reg_reg_s);
12852 %}
12853
12854 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12855 match(Set dst (SubD src1 src2));
12856
12857 ins_cost(INSN_COST * 5);
12858 format %{ "fsubd $dst, $src1, $src2" %}
12859
12860 ins_encode %{
12861 __ fsubd(as_FloatRegister($dst$$reg),
12862 as_FloatRegister($src1$$reg),
12863 as_FloatRegister($src2$$reg));
12864 %}
12865
12866 ins_pipe(fp_dop_reg_reg_d);
12867 %}
12868
12869 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12870 match(Set dst (MulHF src1 src2));
12871 format %{ "fmulh $dst, $src1, $src2" %}
12872 ins_encode %{
12873 __ fmulh($dst$$FloatRegister,
12874 $src1$$FloatRegister,
12875 $src2$$FloatRegister);
12876 %}
12877 ins_pipe(fp_dop_reg_reg_s);
12878 %}
12879
12880 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12881 match(Set dst (MulF src1 src2));
12882
12883 ins_cost(INSN_COST * 6);
12884 format %{ "fmuls $dst, $src1, $src2" %}
12885
12886 ins_encode %{
12887 __ fmuls(as_FloatRegister($dst$$reg),
12888 as_FloatRegister($src1$$reg),
12889 as_FloatRegister($src2$$reg));
12890 %}
12891
12892 ins_pipe(fp_dop_reg_reg_s);
12893 %}
12894
12895 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12896 match(Set dst (MulD src1 src2));
12897
12898 ins_cost(INSN_COST * 6);
12899 format %{ "fmuld $dst, $src1, $src2" %}
12900
12901 ins_encode %{
12902 __ fmuld(as_FloatRegister($dst$$reg),
12903 as_FloatRegister($src1$$reg),
12904 as_FloatRegister($src2$$reg));
12905 %}
12906
12907 ins_pipe(fp_dop_reg_reg_d);
12908 %}
12909
12910 // src1 * src2 + src3 (half-precision float)
12911 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12912 match(Set dst (FmaHF src3 (Binary src1 src2)));
12913 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12914 ins_encode %{
12915 assert(UseFMA, "Needs FMA instructions support.");
12916 __ fmaddh($dst$$FloatRegister,
12917 $src1$$FloatRegister,
12918 $src2$$FloatRegister,
12919 $src3$$FloatRegister);
12920 %}
12921 ins_pipe(pipe_class_default);
12922 %}
12923
12924 // src1 * src2 + src3
12925 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12926 match(Set dst (FmaF src3 (Binary src1 src2)));
12927
12928 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12929
12930 ins_encode %{
12931 assert(UseFMA, "Needs FMA instructions support.");
12932 __ fmadds(as_FloatRegister($dst$$reg),
12933 as_FloatRegister($src1$$reg),
12934 as_FloatRegister($src2$$reg),
12935 as_FloatRegister($src3$$reg));
12936 %}
12937
12938 ins_pipe(pipe_class_default);
12939 %}
12940
12941 // src1 * src2 + src3
12942 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12943 match(Set dst (FmaD src3 (Binary src1 src2)));
12944
12945 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12946
12947 ins_encode %{
12948 assert(UseFMA, "Needs FMA instructions support.");
12949 __ fmaddd(as_FloatRegister($dst$$reg),
12950 as_FloatRegister($src1$$reg),
12951 as_FloatRegister($src2$$reg),
12952 as_FloatRegister($src3$$reg));
12953 %}
12954
12955 ins_pipe(pipe_class_default);
12956 %}
12957
12958 // src1 * (-src2) + src3
12959 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12960 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12961 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12962
12963 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12964
12965 ins_encode %{
12966 assert(UseFMA, "Needs FMA instructions support.");
12967 __ fmsubs(as_FloatRegister($dst$$reg),
12968 as_FloatRegister($src1$$reg),
12969 as_FloatRegister($src2$$reg),
12970 as_FloatRegister($src3$$reg));
12971 %}
12972
12973 ins_pipe(pipe_class_default);
12974 %}
12975
12976 // src1 * (-src2) + src3
12977 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12978 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12979 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12980
12981 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12982
12983 ins_encode %{
12984 assert(UseFMA, "Needs FMA instructions support.");
12985 __ fmsubd(as_FloatRegister($dst$$reg),
12986 as_FloatRegister($src1$$reg),
12987 as_FloatRegister($src2$$reg),
12988 as_FloatRegister($src3$$reg));
12989 %}
12990
12991 ins_pipe(pipe_class_default);
12992 %}
12993
12994 // src1 * (-src2) - src3
12995 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12996 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12997 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12998
12999 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13000
13001 ins_encode %{
13002 assert(UseFMA, "Needs FMA instructions support.");
13003 __ fnmadds(as_FloatRegister($dst$$reg),
13004 as_FloatRegister($src1$$reg),
13005 as_FloatRegister($src2$$reg),
13006 as_FloatRegister($src3$$reg));
13007 %}
13008
13009 ins_pipe(pipe_class_default);
13010 %}
13011
13012 // src1 * (-src2) - src3
13013 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13014 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13015 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13016
13017 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13018
13019 ins_encode %{
13020 assert(UseFMA, "Needs FMA instructions support.");
13021 __ fnmaddd(as_FloatRegister($dst$$reg),
13022 as_FloatRegister($src1$$reg),
13023 as_FloatRegister($src2$$reg),
13024 as_FloatRegister($src3$$reg));
13025 %}
13026
13027 ins_pipe(pipe_class_default);
13028 %}
13029
13030 // src1 * src2 - src3
13031 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13032 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13033
13034 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13035
13036 ins_encode %{
13037 assert(UseFMA, "Needs FMA instructions support.");
13038 __ fnmsubs(as_FloatRegister($dst$$reg),
13039 as_FloatRegister($src1$$reg),
13040 as_FloatRegister($src2$$reg),
13041 as_FloatRegister($src3$$reg));
13042 %}
13043
13044 ins_pipe(pipe_class_default);
13045 %}
13046
13047 // src1 * src2 - src3
13048 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13049 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13050
13051 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13052
13053 ins_encode %{
13054 assert(UseFMA, "Needs FMA instructions support.");
13055 // n.b. insn name should be fnmsubd
13056 __ fnmsub(as_FloatRegister($dst$$reg),
13057 as_FloatRegister($src1$$reg),
13058 as_FloatRegister($src2$$reg),
13059 as_FloatRegister($src3$$reg));
13060 %}
13061
13062 ins_pipe(pipe_class_default);
13063 %}
13064
13065 // Math.max(HH)H (half-precision float)
13066 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13067 match(Set dst (MaxHF src1 src2));
13068 format %{ "fmaxh $dst, $src1, $src2" %}
13069 ins_encode %{
13070 __ fmaxh($dst$$FloatRegister,
13071 $src1$$FloatRegister,
13072 $src2$$FloatRegister);
13073 %}
13074 ins_pipe(fp_dop_reg_reg_s);
13075 %}
13076
13077 // Math.min(HH)H (half-precision float)
13078 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13079 match(Set dst (MinHF src1 src2));
13080 format %{ "fminh $dst, $src1, $src2" %}
13081 ins_encode %{
13082 __ fminh($dst$$FloatRegister,
13083 $src1$$FloatRegister,
13084 $src2$$FloatRegister);
13085 %}
13086 ins_pipe(fp_dop_reg_reg_s);
13087 %}
13088
13089 // Math.max(FF)F
13090 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13091 match(Set dst (MaxF src1 src2));
13092
13093 format %{ "fmaxs $dst, $src1, $src2" %}
13094 ins_encode %{
13095 __ fmaxs(as_FloatRegister($dst$$reg),
13096 as_FloatRegister($src1$$reg),
13097 as_FloatRegister($src2$$reg));
13098 %}
13099
13100 ins_pipe(fp_dop_reg_reg_s);
13101 %}
13102
13103 // Math.min(FF)F
13104 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13105 match(Set dst (MinF src1 src2));
13106
13107 format %{ "fmins $dst, $src1, $src2" %}
13108 ins_encode %{
13109 __ fmins(as_FloatRegister($dst$$reg),
13110 as_FloatRegister($src1$$reg),
13111 as_FloatRegister($src2$$reg));
13112 %}
13113
13114 ins_pipe(fp_dop_reg_reg_s);
13115 %}
13116
13117 // Math.max(DD)D
13118 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13119 match(Set dst (MaxD src1 src2));
13120
13121 format %{ "fmaxd $dst, $src1, $src2" %}
13122 ins_encode %{
13123 __ fmaxd(as_FloatRegister($dst$$reg),
13124 as_FloatRegister($src1$$reg),
13125 as_FloatRegister($src2$$reg));
13126 %}
13127
13128 ins_pipe(fp_dop_reg_reg_d);
13129 %}
13130
13131 // Math.min(DD)D
13132 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13133 match(Set dst (MinD src1 src2));
13134
13135 format %{ "fmind $dst, $src1, $src2" %}
13136 ins_encode %{
13137 __ fmind(as_FloatRegister($dst$$reg),
13138 as_FloatRegister($src1$$reg),
13139 as_FloatRegister($src2$$reg));
13140 %}
13141
13142 ins_pipe(fp_dop_reg_reg_d);
13143 %}
13144
13145 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13146 match(Set dst (DivHF src1 src2));
13147 format %{ "fdivh $dst, $src1, $src2" %}
13148 ins_encode %{
13149 __ fdivh($dst$$FloatRegister,
13150 $src1$$FloatRegister,
13151 $src2$$FloatRegister);
13152 %}
13153 ins_pipe(fp_div_s);
13154 %}
13155
13156 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13157 match(Set dst (DivF src1 src2));
13158
13159 ins_cost(INSN_COST * 18);
13160 format %{ "fdivs $dst, $src1, $src2" %}
13161
13162 ins_encode %{
13163 __ fdivs(as_FloatRegister($dst$$reg),
13164 as_FloatRegister($src1$$reg),
13165 as_FloatRegister($src2$$reg));
13166 %}
13167
13168 ins_pipe(fp_div_s);
13169 %}
13170
13171 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13172 match(Set dst (DivD src1 src2));
13173
13174 ins_cost(INSN_COST * 32);
13175 format %{ "fdivd $dst, $src1, $src2" %}
13176
13177 ins_encode %{
13178 __ fdivd(as_FloatRegister($dst$$reg),
13179 as_FloatRegister($src1$$reg),
13180 as_FloatRegister($src2$$reg));
13181 %}
13182
13183 ins_pipe(fp_div_d);
13184 %}
13185
13186 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13187 match(Set dst (NegF src));
13188
13189 ins_cost(INSN_COST * 3);
13190 format %{ "fneg $dst, $src" %}
13191
13192 ins_encode %{
13193 __ fnegs(as_FloatRegister($dst$$reg),
13194 as_FloatRegister($src$$reg));
13195 %}
13196
13197 ins_pipe(fp_uop_s);
13198 %}
13199
13200 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13201 match(Set dst (NegD src));
13202
13203 ins_cost(INSN_COST * 3);
13204 format %{ "fnegd $dst, $src" %}
13205
13206 ins_encode %{
13207 __ fnegd(as_FloatRegister($dst$$reg),
13208 as_FloatRegister($src$$reg));
13209 %}
13210
13211 ins_pipe(fp_uop_d);
13212 %}
13213
13214 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13215 %{
13216 match(Set dst (AbsI src));
13217
13218 effect(KILL cr);
13219 ins_cost(INSN_COST * 2);
13220 format %{ "cmpw $src, zr\n\t"
13221 "cnegw $dst, $src, Assembler::LT\t# int abs"
13222 %}
13223
13224 ins_encode %{
13225 __ cmpw(as_Register($src$$reg), zr);
13226 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13227 %}
13228 ins_pipe(pipe_class_default);
13229 %}
13230
13231 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13232 %{
13233 match(Set dst (AbsL src));
13234
13235 effect(KILL cr);
13236 ins_cost(INSN_COST * 2);
13237 format %{ "cmp $src, zr\n\t"
13238 "cneg $dst, $src, Assembler::LT\t# long abs"
13239 %}
13240
13241 ins_encode %{
13242 __ cmp(as_Register($src$$reg), zr);
13243 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13244 %}
13245 ins_pipe(pipe_class_default);
13246 %}
13247
13248 instruct absF_reg(vRegF dst, vRegF src) %{
13249 match(Set dst (AbsF src));
13250
13251 ins_cost(INSN_COST * 3);
13252 format %{ "fabss $dst, $src" %}
13253 ins_encode %{
13254 __ fabss(as_FloatRegister($dst$$reg),
13255 as_FloatRegister($src$$reg));
13256 %}
13257
13258 ins_pipe(fp_uop_s);
13259 %}
13260
13261 instruct absD_reg(vRegD dst, vRegD src) %{
13262 match(Set dst (AbsD src));
13263
13264 ins_cost(INSN_COST * 3);
13265 format %{ "fabsd $dst, $src" %}
13266 ins_encode %{
13267 __ fabsd(as_FloatRegister($dst$$reg),
13268 as_FloatRegister($src$$reg));
13269 %}
13270
13271 ins_pipe(fp_uop_d);
13272 %}
13273
13274 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13275 match(Set dst (AbsF (SubF src1 src2)));
13276
13277 ins_cost(INSN_COST * 3);
13278 format %{ "fabds $dst, $src1, $src2" %}
13279 ins_encode %{
13280 __ fabds(as_FloatRegister($dst$$reg),
13281 as_FloatRegister($src1$$reg),
13282 as_FloatRegister($src2$$reg));
13283 %}
13284
13285 ins_pipe(fp_uop_s);
13286 %}
13287
13288 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13289 match(Set dst (AbsD (SubD src1 src2)));
13290
13291 ins_cost(INSN_COST * 3);
13292 format %{ "fabdd $dst, $src1, $src2" %}
13293 ins_encode %{
13294 __ fabdd(as_FloatRegister($dst$$reg),
13295 as_FloatRegister($src1$$reg),
13296 as_FloatRegister($src2$$reg));
13297 %}
13298
13299 ins_pipe(fp_uop_d);
13300 %}
13301
13302 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13303 match(Set dst (SqrtD src));
13304
13305 ins_cost(INSN_COST * 50);
13306 format %{ "fsqrtd $dst, $src" %}
13307 ins_encode %{
13308 __ fsqrtd(as_FloatRegister($dst$$reg),
13309 as_FloatRegister($src$$reg));
13310 %}
13311
13312 ins_pipe(fp_div_s);
13313 %}
13314
13315 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13316 match(Set dst (SqrtF src));
13317
13318 ins_cost(INSN_COST * 50);
13319 format %{ "fsqrts $dst, $src" %}
13320 ins_encode %{
13321 __ fsqrts(as_FloatRegister($dst$$reg),
13322 as_FloatRegister($src$$reg));
13323 %}
13324
13325 ins_pipe(fp_div_d);
13326 %}
13327
13328 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13329 match(Set dst (SqrtHF src));
13330 format %{ "fsqrth $dst, $src" %}
13331 ins_encode %{
13332 __ fsqrth($dst$$FloatRegister,
13333 $src$$FloatRegister);
13334 %}
13335 ins_pipe(fp_div_s);
13336 %}
13337
13338 // Math.rint, floor, ceil
13339 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13340 match(Set dst (RoundDoubleMode src rmode));
13341 format %{ "frint $dst, $src, $rmode" %}
13342 ins_encode %{
13343 switch ($rmode$$constant) {
13344 case RoundDoubleModeNode::rmode_rint:
13345 __ frintnd(as_FloatRegister($dst$$reg),
13346 as_FloatRegister($src$$reg));
13347 break;
13348 case RoundDoubleModeNode::rmode_floor:
13349 __ frintmd(as_FloatRegister($dst$$reg),
13350 as_FloatRegister($src$$reg));
13351 break;
13352 case RoundDoubleModeNode::rmode_ceil:
13353 __ frintpd(as_FloatRegister($dst$$reg),
13354 as_FloatRegister($src$$reg));
13355 break;
13356 }
13357 %}
13358 ins_pipe(fp_uop_d);
13359 %}
13360
13361 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13362 match(Set dst (CopySignD src1 (Binary src2 zero)));
13363 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13364 format %{ "CopySignD $dst $src1 $src2" %}
13365 ins_encode %{
13366 FloatRegister dst = as_FloatRegister($dst$$reg),
13367 src1 = as_FloatRegister($src1$$reg),
13368 src2 = as_FloatRegister($src2$$reg),
13369 zero = as_FloatRegister($zero$$reg);
13370 __ fnegd(dst, zero);
13371 __ bsl(dst, __ T8B, src2, src1);
13372 %}
13373 ins_pipe(fp_uop_d);
13374 %}
13375
13376 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13377 match(Set dst (CopySignF src1 src2));
13378 effect(TEMP_DEF dst, USE src1, USE src2);
13379 format %{ "CopySignF $dst $src1 $src2" %}
13380 ins_encode %{
13381 FloatRegister dst = as_FloatRegister($dst$$reg),
13382 src1 = as_FloatRegister($src1$$reg),
13383 src2 = as_FloatRegister($src2$$reg);
13384 __ movi(dst, __ T2S, 0x80, 24);
13385 __ bsl(dst, __ T8B, src2, src1);
13386 %}
13387 ins_pipe(fp_uop_d);
13388 %}
13389
13390 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13391 match(Set dst (SignumD src (Binary zero one)));
13392 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13393 format %{ "signumD $dst, $src" %}
13394 ins_encode %{
13395 FloatRegister src = as_FloatRegister($src$$reg),
13396 dst = as_FloatRegister($dst$$reg),
13397 zero = as_FloatRegister($zero$$reg),
13398 one = as_FloatRegister($one$$reg);
13399 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13400 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13401 // Bit selection instruction gets bit from "one" for each enabled bit in
13402 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13403 // NaN the whole "src" will be copied because "dst" is zero. For all other
13404 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13405 // from "src", and all other bits are copied from 1.0.
13406 __ bsl(dst, __ T8B, one, src);
13407 %}
13408 ins_pipe(fp_uop_d);
13409 %}
13410
13411 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13412 match(Set dst (SignumF src (Binary zero one)));
13413 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13414 format %{ "signumF $dst, $src" %}
13415 ins_encode %{
13416 FloatRegister src = as_FloatRegister($src$$reg),
13417 dst = as_FloatRegister($dst$$reg),
13418 zero = as_FloatRegister($zero$$reg),
13419 one = as_FloatRegister($one$$reg);
13420 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13421 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13422 // Bit selection instruction gets bit from "one" for each enabled bit in
13423 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13424 // NaN the whole "src" will be copied because "dst" is zero. For all other
13425 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13426 // from "src", and all other bits are copied from 1.0.
13427 __ bsl(dst, __ T8B, one, src);
13428 %}
13429 ins_pipe(fp_uop_d);
13430 %}
13431
13432 instruct onspinwait() %{
13433 match(OnSpinWait);
13434 ins_cost(INSN_COST);
13435
13436 format %{ "onspinwait" %}
13437
13438 ins_encode %{
13439 __ spin_wait();
13440 %}
13441 ins_pipe(pipe_class_empty);
13442 %}
13443
13444 // ============================================================================
13445 // Logical Instructions
13446
13447 // Integer Logical Instructions
13448
13449 // And Instructions
13450
13451
13452 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13453 match(Set dst (AndI src1 src2));
13454
13455 format %{ "andw $dst, $src1, $src2\t# int" %}
13456
13457 ins_cost(INSN_COST);
13458 ins_encode %{
13459 __ andw(as_Register($dst$$reg),
13460 as_Register($src1$$reg),
13461 as_Register($src2$$reg));
13462 %}
13463
13464 ins_pipe(ialu_reg_reg);
13465 %}
13466
13467 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13468 match(Set dst (AndI src1 src2));
13469
13470 format %{ "andsw $dst, $src1, $src2\t# int" %}
13471
13472 ins_cost(INSN_COST);
13473 ins_encode %{
13474 __ andw(as_Register($dst$$reg),
13475 as_Register($src1$$reg),
13476 (uint64_t)($src2$$constant));
13477 %}
13478
13479 ins_pipe(ialu_reg_imm);
13480 %}
13481
13482 // Or Instructions
13483
13484 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13485 match(Set dst (OrI src1 src2));
13486
13487 format %{ "orrw $dst, $src1, $src2\t# int" %}
13488
13489 ins_cost(INSN_COST);
13490 ins_encode %{
13491 __ orrw(as_Register($dst$$reg),
13492 as_Register($src1$$reg),
13493 as_Register($src2$$reg));
13494 %}
13495
13496 ins_pipe(ialu_reg_reg);
13497 %}
13498
13499 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13500 match(Set dst (OrI src1 src2));
13501
13502 format %{ "orrw $dst, $src1, $src2\t# int" %}
13503
13504 ins_cost(INSN_COST);
13505 ins_encode %{
13506 __ orrw(as_Register($dst$$reg),
13507 as_Register($src1$$reg),
13508 (uint64_t)($src2$$constant));
13509 %}
13510
13511 ins_pipe(ialu_reg_imm);
13512 %}
13513
13514 // Xor Instructions
13515
13516 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13517 match(Set dst (XorI src1 src2));
13518
13519 format %{ "eorw $dst, $src1, $src2\t# int" %}
13520
13521 ins_cost(INSN_COST);
13522 ins_encode %{
13523 __ eorw(as_Register($dst$$reg),
13524 as_Register($src1$$reg),
13525 as_Register($src2$$reg));
13526 %}
13527
13528 ins_pipe(ialu_reg_reg);
13529 %}
13530
13531 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13532 match(Set dst (XorI src1 src2));
13533
13534 format %{ "eorw $dst, $src1, $src2\t# int" %}
13535
13536 ins_cost(INSN_COST);
13537 ins_encode %{
13538 __ eorw(as_Register($dst$$reg),
13539 as_Register($src1$$reg),
13540 (uint64_t)($src2$$constant));
13541 %}
13542
13543 ins_pipe(ialu_reg_imm);
13544 %}
13545
13546 // Long Logical Instructions
13547 // TODO
13548
13549 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13550 match(Set dst (AndL src1 src2));
13551
13552 format %{ "and $dst, $src1, $src2\t# int" %}
13553
13554 ins_cost(INSN_COST);
13555 ins_encode %{
13556 __ andr(as_Register($dst$$reg),
13557 as_Register($src1$$reg),
13558 as_Register($src2$$reg));
13559 %}
13560
13561 ins_pipe(ialu_reg_reg);
13562 %}
13563
13564 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13565 match(Set dst (AndL src1 src2));
13566
13567 format %{ "and $dst, $src1, $src2\t# int" %}
13568
13569 ins_cost(INSN_COST);
13570 ins_encode %{
13571 __ andr(as_Register($dst$$reg),
13572 as_Register($src1$$reg),
13573 (uint64_t)($src2$$constant));
13574 %}
13575
13576 ins_pipe(ialu_reg_imm);
13577 %}
13578
13579 // Or Instructions
13580
13581 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13582 match(Set dst (OrL src1 src2));
13583
13584 format %{ "orr $dst, $src1, $src2\t# int" %}
13585
13586 ins_cost(INSN_COST);
13587 ins_encode %{
13588 __ orr(as_Register($dst$$reg),
13589 as_Register($src1$$reg),
13590 as_Register($src2$$reg));
13591 %}
13592
13593 ins_pipe(ialu_reg_reg);
13594 %}
13595
13596 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13597 match(Set dst (OrL src1 src2));
13598
13599 format %{ "orr $dst, $src1, $src2\t# int" %}
13600
13601 ins_cost(INSN_COST);
13602 ins_encode %{
13603 __ orr(as_Register($dst$$reg),
13604 as_Register($src1$$reg),
13605 (uint64_t)($src2$$constant));
13606 %}
13607
13608 ins_pipe(ialu_reg_imm);
13609 %}
13610
13611 // Xor Instructions
13612
13613 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13614 match(Set dst (XorL src1 src2));
13615
13616 format %{ "eor $dst, $src1, $src2\t# int" %}
13617
13618 ins_cost(INSN_COST);
13619 ins_encode %{
13620 __ eor(as_Register($dst$$reg),
13621 as_Register($src1$$reg),
13622 as_Register($src2$$reg));
13623 %}
13624
13625 ins_pipe(ialu_reg_reg);
13626 %}
13627
13628 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13629 match(Set dst (XorL src1 src2));
13630
13631 ins_cost(INSN_COST);
13632 format %{ "eor $dst, $src1, $src2\t# int" %}
13633
13634 ins_encode %{
13635 __ eor(as_Register($dst$$reg),
13636 as_Register($src1$$reg),
13637 (uint64_t)($src2$$constant));
13638 %}
13639
13640 ins_pipe(ialu_reg_imm);
13641 %}
13642
13643 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13644 %{
13645 match(Set dst (ConvI2L src));
13646
13647 ins_cost(INSN_COST);
13648 format %{ "sxtw $dst, $src\t# i2l" %}
13649 ins_encode %{
13650 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13651 %}
13652 ins_pipe(ialu_reg_shift);
13653 %}
13654
13655 // this pattern occurs in bigmath arithmetic
13656 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13657 %{
13658 match(Set dst (AndL (ConvI2L src) mask));
13659
13660 ins_cost(INSN_COST);
13661 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13662 ins_encode %{
13663 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13664 %}
13665
13666 ins_pipe(ialu_reg_shift);
13667 %}
13668
13669 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13670 match(Set dst (ConvL2I src));
13671
13672 ins_cost(INSN_COST);
13673 format %{ "movw $dst, $src \t// l2i" %}
13674
13675 ins_encode %{
13676 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13677 %}
13678
13679 ins_pipe(ialu_reg);
13680 %}
13681
13682 instruct convD2F_reg(vRegF dst, vRegD src) %{
13683 match(Set dst (ConvD2F src));
13684
13685 ins_cost(INSN_COST * 5);
13686 format %{ "fcvtd $dst, $src \t// d2f" %}
13687
13688 ins_encode %{
13689 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13690 %}
13691
13692 ins_pipe(fp_d2f);
13693 %}
13694
13695 instruct convF2D_reg(vRegD dst, vRegF src) %{
13696 match(Set dst (ConvF2D src));
13697
13698 ins_cost(INSN_COST * 5);
13699 format %{ "fcvts $dst, $src \t// f2d" %}
13700
13701 ins_encode %{
13702 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13703 %}
13704
13705 ins_pipe(fp_f2d);
13706 %}
13707
13708 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13709 match(Set dst (ConvF2I src));
13710
13711 ins_cost(INSN_COST * 5);
13712 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13713
13714 ins_encode %{
13715 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13716 %}
13717
13718 ins_pipe(fp_f2i);
13719 %}
13720
13721 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13722 match(Set dst (ConvF2L src));
13723
13724 ins_cost(INSN_COST * 5);
13725 format %{ "fcvtzs $dst, $src \t// f2l" %}
13726
13727 ins_encode %{
13728 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13729 %}
13730
13731 ins_pipe(fp_f2l);
13732 %}
13733
13734 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13735 match(Set dst (ConvF2HF src));
13736 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13737 "smov $dst, $tmp\t# move result from $tmp to $dst"
13738 %}
13739 effect(TEMP tmp);
13740 ins_encode %{
13741 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13742 %}
13743 ins_pipe(pipe_slow);
13744 %}
13745
13746 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13747 match(Set dst (ConvHF2F src));
13748 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13749 "fcvt $dst, $tmp\t# convert half to single precision"
13750 %}
13751 effect(TEMP tmp);
13752 ins_encode %{
13753 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13754 %}
13755 ins_pipe(pipe_slow);
13756 %}
13757
13758 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13759 match(Set dst (ConvI2F src));
13760
13761 ins_cost(INSN_COST * 5);
13762 format %{ "scvtfws $dst, $src \t// i2f" %}
13763
13764 ins_encode %{
13765 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13766 %}
13767
13768 ins_pipe(fp_i2f);
13769 %}
13770
13771 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13772 match(Set dst (ConvL2F src));
13773
13774 ins_cost(INSN_COST * 5);
13775 format %{ "scvtfs $dst, $src \t// l2f" %}
13776
13777 ins_encode %{
13778 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13779 %}
13780
13781 ins_pipe(fp_l2f);
13782 %}
13783
13784 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13785 match(Set dst (ConvD2I src));
13786
13787 ins_cost(INSN_COST * 5);
13788 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13789
13790 ins_encode %{
13791 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13792 %}
13793
13794 ins_pipe(fp_d2i);
13795 %}
13796
13797 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13798 match(Set dst (ConvD2L src));
13799
13800 ins_cost(INSN_COST * 5);
13801 format %{ "fcvtzd $dst, $src \t// d2l" %}
13802
13803 ins_encode %{
13804 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13805 %}
13806
13807 ins_pipe(fp_d2l);
13808 %}
13809
13810 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13811 match(Set dst (ConvI2D src));
13812
13813 ins_cost(INSN_COST * 5);
13814 format %{ "scvtfwd $dst, $src \t// i2d" %}
13815
13816 ins_encode %{
13817 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13818 %}
13819
13820 ins_pipe(fp_i2d);
13821 %}
13822
13823 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13824 match(Set dst (ConvL2D src));
13825
13826 ins_cost(INSN_COST * 5);
13827 format %{ "scvtfd $dst, $src \t// l2d" %}
13828
13829 ins_encode %{
13830 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13831 %}
13832
13833 ins_pipe(fp_l2d);
13834 %}
13835
13836 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13837 %{
13838 match(Set dst (RoundD src));
13839 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13840 format %{ "java_round_double $dst,$src"%}
13841 ins_encode %{
13842 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13843 as_FloatRegister($ftmp$$reg));
13844 %}
13845 ins_pipe(pipe_slow);
13846 %}
13847
13848 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13849 %{
13850 match(Set dst (RoundF src));
13851 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13852 format %{ "java_round_float $dst,$src"%}
13853 ins_encode %{
13854 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13855 as_FloatRegister($ftmp$$reg));
13856 %}
13857 ins_pipe(pipe_slow);
13858 %}
13859
13860 // stack <-> reg and reg <-> reg shuffles with no conversion
13861
13862 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13863
13864 match(Set dst (MoveF2I src));
13865
13866 effect(DEF dst, USE src);
13867
13868 ins_cost(4 * INSN_COST);
13869
13870 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13871
13872 ins_encode %{
13873 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13874 %}
13875
13876 ins_pipe(iload_reg_reg);
13877
13878 %}
13879
13880 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13881
13882 match(Set dst (MoveI2F src));
13883
13884 effect(DEF dst, USE src);
13885
13886 ins_cost(4 * INSN_COST);
13887
13888 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13889
13890 ins_encode %{
13891 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13892 %}
13893
13894 ins_pipe(pipe_class_memory);
13895
13896 %}
13897
13898 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13899
13900 match(Set dst (MoveD2L src));
13901
13902 effect(DEF dst, USE src);
13903
13904 ins_cost(4 * INSN_COST);
13905
13906 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13907
13908 ins_encode %{
13909 __ ldr($dst$$Register, Address(sp, $src$$disp));
13910 %}
13911
13912 ins_pipe(iload_reg_reg);
13913
13914 %}
13915
13916 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13917
13918 match(Set dst (MoveL2D src));
13919
13920 effect(DEF dst, USE src);
13921
13922 ins_cost(4 * INSN_COST);
13923
13924 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13925
13926 ins_encode %{
13927 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13928 %}
13929
13930 ins_pipe(pipe_class_memory);
13931
13932 %}
13933
13934 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13935
13936 match(Set dst (MoveF2I src));
13937
13938 effect(DEF dst, USE src);
13939
13940 ins_cost(INSN_COST);
13941
13942 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13943
13944 ins_encode %{
13945 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13946 %}
13947
13948 ins_pipe(pipe_class_memory);
13949
13950 %}
13951
13952 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13953
13954 match(Set dst (MoveI2F src));
13955
13956 effect(DEF dst, USE src);
13957
13958 ins_cost(INSN_COST);
13959
13960 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13961
13962 ins_encode %{
13963 __ strw($src$$Register, Address(sp, $dst$$disp));
13964 %}
13965
13966 ins_pipe(istore_reg_reg);
13967
13968 %}
13969
13970 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13971
13972 match(Set dst (MoveD2L src));
13973
13974 effect(DEF dst, USE src);
13975
13976 ins_cost(INSN_COST);
13977
13978 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13979
13980 ins_encode %{
13981 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13982 %}
13983
13984 ins_pipe(pipe_class_memory);
13985
13986 %}
13987
13988 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13989
13990 match(Set dst (MoveL2D src));
13991
13992 effect(DEF dst, USE src);
13993
13994 ins_cost(INSN_COST);
13995
13996 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13997
13998 ins_encode %{
13999 __ str($src$$Register, Address(sp, $dst$$disp));
14000 %}
14001
14002 ins_pipe(istore_reg_reg);
14003
14004 %}
14005
14006 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14007
14008 match(Set dst (MoveF2I src));
14009
14010 effect(DEF dst, USE src);
14011
14012 ins_cost(INSN_COST);
14013
14014 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14015
14016 ins_encode %{
14017 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14018 %}
14019
14020 ins_pipe(fp_f2i);
14021
14022 %}
14023
14024 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14025
14026 match(Set dst (MoveI2F src));
14027
14028 effect(DEF dst, USE src);
14029
14030 ins_cost(INSN_COST);
14031
14032 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14033
14034 ins_encode %{
14035 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14036 %}
14037
14038 ins_pipe(fp_i2f);
14039
14040 %}
14041
14042 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14043
14044 match(Set dst (MoveD2L src));
14045
14046 effect(DEF dst, USE src);
14047
14048 ins_cost(INSN_COST);
14049
14050 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14051
14052 ins_encode %{
14053 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14054 %}
14055
14056 ins_pipe(fp_d2l);
14057
14058 %}
14059
14060 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14061
14062 match(Set dst (MoveL2D src));
14063
14064 effect(DEF dst, USE src);
14065
14066 ins_cost(INSN_COST);
14067
14068 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14069
14070 ins_encode %{
14071 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14072 %}
14073
14074 ins_pipe(fp_l2d);
14075
14076 %}
14077
14078 // ============================================================================
14079 // clearing of an array
14080
14081 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14082 %{
14083 match(Set dummy (ClearArray cnt base));
14084 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14085
14086 ins_cost(4 * INSN_COST);
14087 format %{ "ClearArray $cnt, $base" %}
14088
14089 ins_encode %{
14090 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14091 if (tpc == nullptr) {
14092 ciEnv::current()->record_failure("CodeCache is full");
14093 return;
14094 }
14095 %}
14096
14097 ins_pipe(pipe_class_memory);
14098 %}
14099
14100 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14101 %{
14102 predicate((uint64_t)n->in(2)->get_long()
14103 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14104 match(Set dummy (ClearArray cnt base));
14105 effect(TEMP temp, USE_KILL base, KILL cr);
14106
14107 ins_cost(4 * INSN_COST);
14108 format %{ "ClearArray $cnt, $base" %}
14109
14110 ins_encode %{
14111 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14112 if (tpc == nullptr) {
14113 ciEnv::current()->record_failure("CodeCache is full");
14114 return;
14115 }
14116 %}
14117
14118 ins_pipe(pipe_class_memory);
14119 %}
14120
14121 // ============================================================================
14122 // Overflow Math Instructions
14123
14124 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14125 %{
14126 match(Set cr (OverflowAddI op1 op2));
14127
14128 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14129 ins_cost(INSN_COST);
14130 ins_encode %{
14131 __ cmnw($op1$$Register, $op2$$Register);
14132 %}
14133
14134 ins_pipe(icmp_reg_reg);
14135 %}
14136
14137 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14138 %{
14139 match(Set cr (OverflowAddI op1 op2));
14140
14141 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14142 ins_cost(INSN_COST);
14143 ins_encode %{
14144 __ cmnw($op1$$Register, $op2$$constant);
14145 %}
14146
14147 ins_pipe(icmp_reg_imm);
14148 %}
14149
14150 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14151 %{
14152 match(Set cr (OverflowAddL op1 op2));
14153
14154 format %{ "cmn $op1, $op2\t# overflow check long" %}
14155 ins_cost(INSN_COST);
14156 ins_encode %{
14157 __ cmn($op1$$Register, $op2$$Register);
14158 %}
14159
14160 ins_pipe(icmp_reg_reg);
14161 %}
14162
14163 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14164 %{
14165 match(Set cr (OverflowAddL op1 op2));
14166
14167 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14168 ins_cost(INSN_COST);
14169 ins_encode %{
14170 __ adds(zr, $op1$$Register, $op2$$constant);
14171 %}
14172
14173 ins_pipe(icmp_reg_imm);
14174 %}
14175
14176 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14177 %{
14178 match(Set cr (OverflowSubI op1 op2));
14179
14180 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14181 ins_cost(INSN_COST);
14182 ins_encode %{
14183 __ cmpw($op1$$Register, $op2$$Register);
14184 %}
14185
14186 ins_pipe(icmp_reg_reg);
14187 %}
14188
14189 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14190 %{
14191 match(Set cr (OverflowSubI op1 op2));
14192
14193 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14194 ins_cost(INSN_COST);
14195 ins_encode %{
14196 __ cmpw($op1$$Register, $op2$$constant);
14197 %}
14198
14199 ins_pipe(icmp_reg_imm);
14200 %}
14201
14202 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14203 %{
14204 match(Set cr (OverflowSubL op1 op2));
14205
14206 format %{ "cmp $op1, $op2\t# overflow check long" %}
14207 ins_cost(INSN_COST);
14208 ins_encode %{
14209 __ cmp($op1$$Register, $op2$$Register);
14210 %}
14211
14212 ins_pipe(icmp_reg_reg);
14213 %}
14214
14215 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14216 %{
14217 match(Set cr (OverflowSubL op1 op2));
14218
14219 format %{ "cmp $op1, $op2\t# overflow check long" %}
14220 ins_cost(INSN_COST);
14221 ins_encode %{
14222 __ subs(zr, $op1$$Register, $op2$$constant);
14223 %}
14224
14225 ins_pipe(icmp_reg_imm);
14226 %}
14227
14228 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14229 %{
14230 match(Set cr (OverflowSubI zero op1));
14231
14232 format %{ "cmpw zr, $op1\t# overflow check int" %}
14233 ins_cost(INSN_COST);
14234 ins_encode %{
14235 __ cmpw(zr, $op1$$Register);
14236 %}
14237
14238 ins_pipe(icmp_reg_imm);
14239 %}
14240
14241 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14242 %{
14243 match(Set cr (OverflowSubL zero op1));
14244
14245 format %{ "cmp zr, $op1\t# overflow check long" %}
14246 ins_cost(INSN_COST);
14247 ins_encode %{
14248 __ cmp(zr, $op1$$Register);
14249 %}
14250
14251 ins_pipe(icmp_reg_imm);
14252 %}
14253
14254 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14255 %{
14256 match(Set cr (OverflowMulI op1 op2));
14257
14258 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14259 "cmp rscratch1, rscratch1, sxtw\n\t"
14260 "movw rscratch1, #0x80000000\n\t"
14261 "cselw rscratch1, rscratch1, zr, NE\n\t"
14262 "cmpw rscratch1, #1" %}
14263 ins_cost(5 * INSN_COST);
14264 ins_encode %{
14265 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14266 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14267 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14268 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14269 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14270 %}
14271
14272 ins_pipe(pipe_slow);
14273 %}
14274
14275 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14276 %{
14277 match(If cmp (OverflowMulI op1 op2));
14278 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14279 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14280 effect(USE labl, KILL cr);
14281
14282 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14283 "cmp rscratch1, rscratch1, sxtw\n\t"
14284 "b$cmp $labl" %}
14285 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14286 ins_encode %{
14287 Label* L = $labl$$label;
14288 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14289 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14290 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14291 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14292 %}
14293
14294 ins_pipe(pipe_serial);
14295 %}
14296
14297 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14298 %{
14299 match(Set cr (OverflowMulL op1 op2));
14300
14301 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14302 "smulh rscratch2, $op1, $op2\n\t"
14303 "cmp rscratch2, rscratch1, ASR #63\n\t"
14304 "movw rscratch1, #0x80000000\n\t"
14305 "cselw rscratch1, rscratch1, zr, NE\n\t"
14306 "cmpw rscratch1, #1" %}
14307 ins_cost(6 * INSN_COST);
14308 ins_encode %{
14309 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14310 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14311 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14312 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14313 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14314 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14315 %}
14316
14317 ins_pipe(pipe_slow);
14318 %}
14319
14320 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14321 %{
14322 match(If cmp (OverflowMulL op1 op2));
14323 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14324 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14325 effect(USE labl, KILL cr);
14326
14327 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14328 "smulh rscratch2, $op1, $op2\n\t"
14329 "cmp rscratch2, rscratch1, ASR #63\n\t"
14330 "b$cmp $labl" %}
14331 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14332 ins_encode %{
14333 Label* L = $labl$$label;
14334 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14335 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14336 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14337 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14338 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14339 %}
14340
14341 ins_pipe(pipe_serial);
14342 %}
14343
14344 // ============================================================================
14345 // Compare Instructions
14346
14347 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14348 %{
14349 match(Set cr (CmpI op1 op2));
14350
14351 effect(DEF cr, USE op1, USE op2);
14352
14353 ins_cost(INSN_COST);
14354 format %{ "cmpw $op1, $op2" %}
14355
14356 ins_encode(aarch64_enc_cmpw(op1, op2));
14357
14358 ins_pipe(icmp_reg_reg);
14359 %}
14360
14361 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14362 %{
14363 match(Set cr (CmpI op1 zero));
14364
14365 effect(DEF cr, USE op1);
14366
14367 ins_cost(INSN_COST);
14368 format %{ "cmpw $op1, 0" %}
14369
14370 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14371
14372 ins_pipe(icmp_reg_imm);
14373 %}
14374
14375 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14376 %{
14377 match(Set cr (CmpI op1 op2));
14378
14379 effect(DEF cr, USE op1);
14380
14381 ins_cost(INSN_COST);
14382 format %{ "cmpw $op1, $op2" %}
14383
14384 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14385
14386 ins_pipe(icmp_reg_imm);
14387 %}
14388
14389 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14390 %{
14391 match(Set cr (CmpI op1 op2));
14392
14393 effect(DEF cr, USE op1);
14394
14395 ins_cost(INSN_COST * 2);
14396 format %{ "cmpw $op1, $op2" %}
14397
14398 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14399
14400 ins_pipe(icmp_reg_imm);
14401 %}
14402
14403 // Unsigned compare Instructions; really, same as signed compare
14404 // except it should only be used to feed an If or a CMovI which takes a
14405 // cmpOpU.
14406
14407 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14408 %{
14409 match(Set cr (CmpU op1 op2));
14410
14411 effect(DEF cr, USE op1, USE op2);
14412
14413 ins_cost(INSN_COST);
14414 format %{ "cmpw $op1, $op2\t# unsigned" %}
14415
14416 ins_encode(aarch64_enc_cmpw(op1, op2));
14417
14418 ins_pipe(icmp_reg_reg);
14419 %}
14420
14421 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14422 %{
14423 match(Set cr (CmpU op1 zero));
14424
14425 effect(DEF cr, USE op1);
14426
14427 ins_cost(INSN_COST);
14428 format %{ "cmpw $op1, #0\t# unsigned" %}
14429
14430 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14431
14432 ins_pipe(icmp_reg_imm);
14433 %}
14434
14435 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14436 %{
14437 match(Set cr (CmpU op1 op2));
14438
14439 effect(DEF cr, USE op1);
14440
14441 ins_cost(INSN_COST);
14442 format %{ "cmpw $op1, $op2\t# unsigned" %}
14443
14444 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14445
14446 ins_pipe(icmp_reg_imm);
14447 %}
14448
14449 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14450 %{
14451 match(Set cr (CmpU op1 op2));
14452
14453 effect(DEF cr, USE op1);
14454
14455 ins_cost(INSN_COST * 2);
14456 format %{ "cmpw $op1, $op2\t# unsigned" %}
14457
14458 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14459
14460 ins_pipe(icmp_reg_imm);
14461 %}
14462
14463 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14464 %{
14465 match(Set cr (CmpL op1 op2));
14466
14467 effect(DEF cr, USE op1, USE op2);
14468
14469 ins_cost(INSN_COST);
14470 format %{ "cmp $op1, $op2" %}
14471
14472 ins_encode(aarch64_enc_cmp(op1, op2));
14473
14474 ins_pipe(icmp_reg_reg);
14475 %}
14476
14477 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14478 %{
14479 match(Set cr (CmpL op1 zero));
14480
14481 effect(DEF cr, USE op1);
14482
14483 ins_cost(INSN_COST);
14484 format %{ "tst $op1" %}
14485
14486 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14487
14488 ins_pipe(icmp_reg_imm);
14489 %}
14490
14491 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14492 %{
14493 match(Set cr (CmpL op1 op2));
14494
14495 effect(DEF cr, USE op1);
14496
14497 ins_cost(INSN_COST);
14498 format %{ "cmp $op1, $op2" %}
14499
14500 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14501
14502 ins_pipe(icmp_reg_imm);
14503 %}
14504
14505 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14506 %{
14507 match(Set cr (CmpL op1 op2));
14508
14509 effect(DEF cr, USE op1);
14510
14511 ins_cost(INSN_COST * 2);
14512 format %{ "cmp $op1, $op2" %}
14513
14514 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14515
14516 ins_pipe(icmp_reg_imm);
14517 %}
14518
14519 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14520 %{
14521 match(Set cr (CmpUL op1 op2));
14522
14523 effect(DEF cr, USE op1, USE op2);
14524
14525 ins_cost(INSN_COST);
14526 format %{ "cmp $op1, $op2" %}
14527
14528 ins_encode(aarch64_enc_cmp(op1, op2));
14529
14530 ins_pipe(icmp_reg_reg);
14531 %}
14532
14533 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14534 %{
14535 match(Set cr (CmpUL op1 zero));
14536
14537 effect(DEF cr, USE op1);
14538
14539 ins_cost(INSN_COST);
14540 format %{ "tst $op1" %}
14541
14542 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14543
14544 ins_pipe(icmp_reg_imm);
14545 %}
14546
14547 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14548 %{
14549 match(Set cr (CmpUL op1 op2));
14550
14551 effect(DEF cr, USE op1);
14552
14553 ins_cost(INSN_COST);
14554 format %{ "cmp $op1, $op2" %}
14555
14556 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14557
14558 ins_pipe(icmp_reg_imm);
14559 %}
14560
14561 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14562 %{
14563 match(Set cr (CmpUL op1 op2));
14564
14565 effect(DEF cr, USE op1);
14566
14567 ins_cost(INSN_COST * 2);
14568 format %{ "cmp $op1, $op2" %}
14569
14570 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14571
14572 ins_pipe(icmp_reg_imm);
14573 %}
14574
14575 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14576 %{
14577 match(Set cr (CmpP op1 op2));
14578
14579 effect(DEF cr, USE op1, USE op2);
14580
14581 ins_cost(INSN_COST);
14582 format %{ "cmp $op1, $op2\t // ptr" %}
14583
14584 ins_encode(aarch64_enc_cmpp(op1, op2));
14585
14586 ins_pipe(icmp_reg_reg);
14587 %}
14588
14589 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14590 %{
14591 match(Set cr (CmpN op1 op2));
14592
14593 effect(DEF cr, USE op1, USE op2);
14594
14595 ins_cost(INSN_COST);
14596 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14597
14598 ins_encode(aarch64_enc_cmpn(op1, op2));
14599
14600 ins_pipe(icmp_reg_reg);
14601 %}
14602
14603 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14604 %{
14605 match(Set cr (CmpP op1 zero));
14606
14607 effect(DEF cr, USE op1, USE zero);
14608
14609 ins_cost(INSN_COST);
14610 format %{ "cmp $op1, 0\t // ptr" %}
14611
14612 ins_encode(aarch64_enc_testp(op1));
14613
14614 ins_pipe(icmp_reg_imm);
14615 %}
14616
14617 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14618 %{
14619 match(Set cr (CmpN op1 zero));
14620
14621 effect(DEF cr, USE op1, USE zero);
14622
14623 ins_cost(INSN_COST);
14624 format %{ "cmp $op1, 0\t // compressed ptr" %}
14625
14626 ins_encode(aarch64_enc_testn(op1));
14627
14628 ins_pipe(icmp_reg_imm);
14629 %}
14630
14631 // FP comparisons
14632 //
14633 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14634 // using normal cmpOp. See declaration of rFlagsReg for details.
14635
14636 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14637 %{
14638 match(Set cr (CmpF src1 src2));
14639
14640 ins_cost(3 * INSN_COST);
14641 format %{ "fcmps $src1, $src2" %}
14642
14643 ins_encode %{
14644 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14645 %}
14646
14647 ins_pipe(pipe_class_compare);
14648 %}
14649
14650 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14651 %{
14652 match(Set cr (CmpF src1 src2));
14653
14654 ins_cost(3 * INSN_COST);
14655 format %{ "fcmps $src1, 0.0" %}
14656
14657 ins_encode %{
14658 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14659 %}
14660
14661 ins_pipe(pipe_class_compare);
14662 %}
14663 // FROM HERE
14664
14665 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14666 %{
14667 match(Set cr (CmpD src1 src2));
14668
14669 ins_cost(3 * INSN_COST);
14670 format %{ "fcmpd $src1, $src2" %}
14671
14672 ins_encode %{
14673 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14674 %}
14675
14676 ins_pipe(pipe_class_compare);
14677 %}
14678
14679 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14680 %{
14681 match(Set cr (CmpD src1 src2));
14682
14683 ins_cost(3 * INSN_COST);
14684 format %{ "fcmpd $src1, 0.0" %}
14685
14686 ins_encode %{
14687 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14688 %}
14689
14690 ins_pipe(pipe_class_compare);
14691 %}
14692
14693 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14694 %{
14695 match(Set dst (CmpF3 src1 src2));
14696 effect(KILL cr);
14697
14698 ins_cost(5 * INSN_COST);
14699 format %{ "fcmps $src1, $src2\n\t"
14700 "csinvw($dst, zr, zr, eq\n\t"
14701 "csnegw($dst, $dst, $dst, lt)"
14702 %}
14703
14704 ins_encode %{
14705 Label done;
14706 FloatRegister s1 = as_FloatRegister($src1$$reg);
14707 FloatRegister s2 = as_FloatRegister($src2$$reg);
14708 Register d = as_Register($dst$$reg);
14709 __ fcmps(s1, s2);
14710 // installs 0 if EQ else -1
14711 __ csinvw(d, zr, zr, Assembler::EQ);
14712 // keeps -1 if less or unordered else installs 1
14713 __ csnegw(d, d, d, Assembler::LT);
14714 __ bind(done);
14715 %}
14716
14717 ins_pipe(pipe_class_default);
14718
14719 %}
14720
14721 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14722 %{
14723 match(Set dst (CmpD3 src1 src2));
14724 effect(KILL cr);
14725
14726 ins_cost(5 * INSN_COST);
14727 format %{ "fcmpd $src1, $src2\n\t"
14728 "csinvw($dst, zr, zr, eq\n\t"
14729 "csnegw($dst, $dst, $dst, lt)"
14730 %}
14731
14732 ins_encode %{
14733 Label done;
14734 FloatRegister s1 = as_FloatRegister($src1$$reg);
14735 FloatRegister s2 = as_FloatRegister($src2$$reg);
14736 Register d = as_Register($dst$$reg);
14737 __ fcmpd(s1, s2);
14738 // installs 0 if EQ else -1
14739 __ csinvw(d, zr, zr, Assembler::EQ);
14740 // keeps -1 if less or unordered else installs 1
14741 __ csnegw(d, d, d, Assembler::LT);
14742 __ bind(done);
14743 %}
14744 ins_pipe(pipe_class_default);
14745
14746 %}
14747
14748 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14749 %{
14750 match(Set dst (CmpF3 src1 zero));
14751 effect(KILL cr);
14752
14753 ins_cost(5 * INSN_COST);
14754 format %{ "fcmps $src1, 0.0\n\t"
14755 "csinvw($dst, zr, zr, eq\n\t"
14756 "csnegw($dst, $dst, $dst, lt)"
14757 %}
14758
14759 ins_encode %{
14760 Label done;
14761 FloatRegister s1 = as_FloatRegister($src1$$reg);
14762 Register d = as_Register($dst$$reg);
14763 __ fcmps(s1, 0.0);
14764 // installs 0 if EQ else -1
14765 __ csinvw(d, zr, zr, Assembler::EQ);
14766 // keeps -1 if less or unordered else installs 1
14767 __ csnegw(d, d, d, Assembler::LT);
14768 __ bind(done);
14769 %}
14770
14771 ins_pipe(pipe_class_default);
14772
14773 %}
14774
14775 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14776 %{
14777 match(Set dst (CmpD3 src1 zero));
14778 effect(KILL cr);
14779
14780 ins_cost(5 * INSN_COST);
14781 format %{ "fcmpd $src1, 0.0\n\t"
14782 "csinvw($dst, zr, zr, eq\n\t"
14783 "csnegw($dst, $dst, $dst, lt)"
14784 %}
14785
14786 ins_encode %{
14787 Label done;
14788 FloatRegister s1 = as_FloatRegister($src1$$reg);
14789 Register d = as_Register($dst$$reg);
14790 __ fcmpd(s1, 0.0);
14791 // installs 0 if EQ else -1
14792 __ csinvw(d, zr, zr, Assembler::EQ);
14793 // keeps -1 if less or unordered else installs 1
14794 __ csnegw(d, d, d, Assembler::LT);
14795 __ bind(done);
14796 %}
14797 ins_pipe(pipe_class_default);
14798
14799 %}
14800
14801 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14802 %{
14803 match(Set dst (CmpLTMask p q));
14804 effect(KILL cr);
14805
14806 ins_cost(3 * INSN_COST);
14807
14808 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14809 "csetw $dst, lt\n\t"
14810 "subw $dst, zr, $dst"
14811 %}
14812
14813 ins_encode %{
14814 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14815 __ csetw(as_Register($dst$$reg), Assembler::LT);
14816 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14817 %}
14818
14819 ins_pipe(ialu_reg_reg);
14820 %}
14821
14822 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14823 %{
14824 match(Set dst (CmpLTMask src zero));
14825 effect(KILL cr);
14826
14827 ins_cost(INSN_COST);
14828
14829 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14830
14831 ins_encode %{
14832 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14833 %}
14834
14835 ins_pipe(ialu_reg_shift);
14836 %}
14837
14838 // ============================================================================
14839 // Max and Min
14840
14841 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14842
14843 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14844 %{
14845 effect(DEF cr, USE src);
14846 ins_cost(INSN_COST);
14847 format %{ "cmpw $src, 0" %}
14848
14849 ins_encode %{
14850 __ cmpw($src$$Register, 0);
14851 %}
14852 ins_pipe(icmp_reg_imm);
14853 %}
14854
14855 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14856 %{
14857 match(Set dst (MinI src1 src2));
14858 ins_cost(INSN_COST * 3);
14859
14860 expand %{
14861 rFlagsReg cr;
14862 compI_reg_reg(cr, src1, src2);
14863 cmovI_reg_reg_lt(dst, src1, src2, cr);
14864 %}
14865 %}
14866
14867 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14868 %{
14869 match(Set dst (MaxI src1 src2));
14870 ins_cost(INSN_COST * 3);
14871
14872 expand %{
14873 rFlagsReg cr;
14874 compI_reg_reg(cr, src1, src2);
14875 cmovI_reg_reg_gt(dst, src1, src2, cr);
14876 %}
14877 %}
14878
14879
14880 // ============================================================================
14881 // Branch Instructions
14882
14883 // Direct Branch.
14884 instruct branch(label lbl)
14885 %{
14886 match(Goto);
14887
14888 effect(USE lbl);
14889
14890 ins_cost(BRANCH_COST);
14891 format %{ "b $lbl" %}
14892
14893 ins_encode(aarch64_enc_b(lbl));
14894
14895 ins_pipe(pipe_branch);
14896 %}
14897
14898 // Conditional Near Branch
14899 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14900 %{
14901 // Same match rule as `branchConFar'.
14902 match(If cmp cr);
14903
14904 effect(USE lbl);
14905
14906 ins_cost(BRANCH_COST);
14907 // If set to 1 this indicates that the current instruction is a
14908 // short variant of a long branch. This avoids using this
14909 // instruction in first-pass matching. It will then only be used in
14910 // the `Shorten_branches' pass.
14911 // ins_short_branch(1);
14912 format %{ "b$cmp $lbl" %}
14913
14914 ins_encode(aarch64_enc_br_con(cmp, lbl));
14915
14916 ins_pipe(pipe_branch_cond);
14917 %}
14918
14919 // Conditional Near Branch Unsigned
14920 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14921 %{
14922 // Same match rule as `branchConFar'.
14923 match(If cmp cr);
14924
14925 effect(USE lbl);
14926
14927 ins_cost(BRANCH_COST);
14928 // If set to 1 this indicates that the current instruction is a
14929 // short variant of a long branch. This avoids using this
14930 // instruction in first-pass matching. It will then only be used in
14931 // the `Shorten_branches' pass.
14932 // ins_short_branch(1);
14933 format %{ "b$cmp $lbl\t# unsigned" %}
14934
14935 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14936
14937 ins_pipe(pipe_branch_cond);
14938 %}
14939
14940 // Make use of CBZ and CBNZ. These instructions, as well as being
14941 // shorter than (cmp; branch), have the additional benefit of not
14942 // killing the flags.
14943
14944 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14945 match(If cmp (CmpI op1 op2));
14946 effect(USE labl);
14947
14948 ins_cost(BRANCH_COST);
14949 format %{ "cbw$cmp $op1, $labl" %}
14950 ins_encode %{
14951 Label* L = $labl$$label;
14952 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14953 if (cond == Assembler::EQ)
14954 __ cbzw($op1$$Register, *L);
14955 else
14956 __ cbnzw($op1$$Register, *L);
14957 %}
14958 ins_pipe(pipe_cmp_branch);
14959 %}
14960
14961 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14962 match(If cmp (CmpL op1 op2));
14963 effect(USE labl);
14964
14965 ins_cost(BRANCH_COST);
14966 format %{ "cb$cmp $op1, $labl" %}
14967 ins_encode %{
14968 Label* L = $labl$$label;
14969 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14970 if (cond == Assembler::EQ)
14971 __ cbz($op1$$Register, *L);
14972 else
14973 __ cbnz($op1$$Register, *L);
14974 %}
14975 ins_pipe(pipe_cmp_branch);
14976 %}
14977
14978 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14979 match(If cmp (CmpP op1 op2));
14980 effect(USE labl);
14981
14982 ins_cost(BRANCH_COST);
14983 format %{ "cb$cmp $op1, $labl" %}
14984 ins_encode %{
14985 Label* L = $labl$$label;
14986 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14987 if (cond == Assembler::EQ)
14988 __ cbz($op1$$Register, *L);
14989 else
14990 __ cbnz($op1$$Register, *L);
14991 %}
14992 ins_pipe(pipe_cmp_branch);
14993 %}
14994
14995 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14996 match(If cmp (CmpN op1 op2));
14997 effect(USE labl);
14998
14999 ins_cost(BRANCH_COST);
15000 format %{ "cbw$cmp $op1, $labl" %}
15001 ins_encode %{
15002 Label* L = $labl$$label;
15003 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15004 if (cond == Assembler::EQ)
15005 __ cbzw($op1$$Register, *L);
15006 else
15007 __ cbnzw($op1$$Register, *L);
15008 %}
15009 ins_pipe(pipe_cmp_branch);
15010 %}
15011
15012 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15013 match(If cmp (CmpP (DecodeN oop) zero));
15014 effect(USE labl);
15015
15016 ins_cost(BRANCH_COST);
15017 format %{ "cb$cmp $oop, $labl" %}
15018 ins_encode %{
15019 Label* L = $labl$$label;
15020 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15021 if (cond == Assembler::EQ)
15022 __ cbzw($oop$$Register, *L);
15023 else
15024 __ cbnzw($oop$$Register, *L);
15025 %}
15026 ins_pipe(pipe_cmp_branch);
15027 %}
15028
15029 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15030 match(If cmp (CmpU op1 op2));
15031 effect(USE labl);
15032
15033 ins_cost(BRANCH_COST);
15034 format %{ "cbw$cmp $op1, $labl" %}
15035 ins_encode %{
15036 Label* L = $labl$$label;
15037 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15038 if (cond == Assembler::EQ || cond == Assembler::LS) {
15039 __ cbzw($op1$$Register, *L);
15040 } else {
15041 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15042 __ cbnzw($op1$$Register, *L);
15043 }
15044 %}
15045 ins_pipe(pipe_cmp_branch);
15046 %}
15047
15048 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15049 match(If cmp (CmpUL op1 op2));
15050 effect(USE labl);
15051
15052 ins_cost(BRANCH_COST);
15053 format %{ "cb$cmp $op1, $labl" %}
15054 ins_encode %{
15055 Label* L = $labl$$label;
15056 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15057 if (cond == Assembler::EQ || cond == Assembler::LS) {
15058 __ cbz($op1$$Register, *L);
15059 } else {
15060 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15061 __ cbnz($op1$$Register, *L);
15062 }
15063 %}
15064 ins_pipe(pipe_cmp_branch);
15065 %}
15066
15067 // Test bit and Branch
15068
15069 // Patterns for short (< 32KiB) variants
15070 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15071 match(If cmp (CmpL op1 op2));
15072 effect(USE labl);
15073
15074 ins_cost(BRANCH_COST);
15075 format %{ "cb$cmp $op1, $labl # long" %}
15076 ins_encode %{
15077 Label* L = $labl$$label;
15078 Assembler::Condition cond =
15079 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15080 __ tbr(cond, $op1$$Register, 63, *L);
15081 %}
15082 ins_pipe(pipe_cmp_branch);
15083 ins_short_branch(1);
15084 %}
15085
15086 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15087 match(If cmp (CmpI op1 op2));
15088 effect(USE labl);
15089
15090 ins_cost(BRANCH_COST);
15091 format %{ "cb$cmp $op1, $labl # int" %}
15092 ins_encode %{
15093 Label* L = $labl$$label;
15094 Assembler::Condition cond =
15095 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15096 __ tbr(cond, $op1$$Register, 31, *L);
15097 %}
15098 ins_pipe(pipe_cmp_branch);
15099 ins_short_branch(1);
15100 %}
15101
15102 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15103 match(If cmp (CmpL (AndL op1 op2) op3));
15104 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15105 effect(USE labl);
15106
15107 ins_cost(BRANCH_COST);
15108 format %{ "tb$cmp $op1, $op2, $labl" %}
15109 ins_encode %{
15110 Label* L = $labl$$label;
15111 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15112 int bit = exact_log2_long($op2$$constant);
15113 __ tbr(cond, $op1$$Register, bit, *L);
15114 %}
15115 ins_pipe(pipe_cmp_branch);
15116 ins_short_branch(1);
15117 %}
15118
15119 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15120 match(If cmp (CmpI (AndI op1 op2) op3));
15121 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15122 effect(USE labl);
15123
15124 ins_cost(BRANCH_COST);
15125 format %{ "tb$cmp $op1, $op2, $labl" %}
15126 ins_encode %{
15127 Label* L = $labl$$label;
15128 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15129 int bit = exact_log2((juint)$op2$$constant);
15130 __ tbr(cond, $op1$$Register, bit, *L);
15131 %}
15132 ins_pipe(pipe_cmp_branch);
15133 ins_short_branch(1);
15134 %}
15135
15136 // And far variants
15137 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15138 match(If cmp (CmpL op1 op2));
15139 effect(USE labl);
15140
15141 ins_cost(BRANCH_COST);
15142 format %{ "cb$cmp $op1, $labl # long" %}
15143 ins_encode %{
15144 Label* L = $labl$$label;
15145 Assembler::Condition cond =
15146 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15147 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15148 %}
15149 ins_pipe(pipe_cmp_branch);
15150 %}
15151
15152 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15153 match(If cmp (CmpI op1 op2));
15154 effect(USE labl);
15155
15156 ins_cost(BRANCH_COST);
15157 format %{ "cb$cmp $op1, $labl # int" %}
15158 ins_encode %{
15159 Label* L = $labl$$label;
15160 Assembler::Condition cond =
15161 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15162 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15163 %}
15164 ins_pipe(pipe_cmp_branch);
15165 %}
15166
15167 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15168 match(If cmp (CmpL (AndL op1 op2) op3));
15169 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15170 effect(USE labl);
15171
15172 ins_cost(BRANCH_COST);
15173 format %{ "tb$cmp $op1, $op2, $labl" %}
15174 ins_encode %{
15175 Label* L = $labl$$label;
15176 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15177 int bit = exact_log2_long($op2$$constant);
15178 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15179 %}
15180 ins_pipe(pipe_cmp_branch);
15181 %}
15182
15183 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15184 match(If cmp (CmpI (AndI op1 op2) op3));
15185 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15186 effect(USE labl);
15187
15188 ins_cost(BRANCH_COST);
15189 format %{ "tb$cmp $op1, $op2, $labl" %}
15190 ins_encode %{
15191 Label* L = $labl$$label;
15192 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15193 int bit = exact_log2((juint)$op2$$constant);
15194 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15195 %}
15196 ins_pipe(pipe_cmp_branch);
15197 %}
15198
15199 // Test bits
15200
15201 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15202 match(Set cr (CmpL (AndL op1 op2) op3));
15203 predicate(Assembler::operand_valid_for_logical_immediate
15204 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15205
15206 ins_cost(INSN_COST);
15207 format %{ "tst $op1, $op2 # long" %}
15208 ins_encode %{
15209 __ tst($op1$$Register, $op2$$constant);
15210 %}
15211 ins_pipe(ialu_reg_reg);
15212 %}
15213
15214 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15215 match(Set cr (CmpI (AndI op1 op2) op3));
15216 predicate(Assembler::operand_valid_for_logical_immediate
15217 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15218
15219 ins_cost(INSN_COST);
15220 format %{ "tst $op1, $op2 # int" %}
15221 ins_encode %{
15222 __ tstw($op1$$Register, $op2$$constant);
15223 %}
15224 ins_pipe(ialu_reg_reg);
15225 %}
15226
15227 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15228 match(Set cr (CmpL (AndL op1 op2) op3));
15229
15230 ins_cost(INSN_COST);
15231 format %{ "tst $op1, $op2 # long" %}
15232 ins_encode %{
15233 __ tst($op1$$Register, $op2$$Register);
15234 %}
15235 ins_pipe(ialu_reg_reg);
15236 %}
15237
15238 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15239 match(Set cr (CmpI (AndI op1 op2) op3));
15240
15241 ins_cost(INSN_COST);
15242 format %{ "tstw $op1, $op2 # int" %}
15243 ins_encode %{
15244 __ tstw($op1$$Register, $op2$$Register);
15245 %}
15246 ins_pipe(ialu_reg_reg);
15247 %}
15248
15249
15250 // Conditional Far Branch
15251 // Conditional Far Branch Unsigned
15252 // TODO: fixme
15253
15254 // counted loop end branch near
15255 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15256 %{
15257 match(CountedLoopEnd cmp cr);
15258
15259 effect(USE lbl);
15260
15261 ins_cost(BRANCH_COST);
15262 // short variant.
15263 // ins_short_branch(1);
15264 format %{ "b$cmp $lbl \t// counted loop end" %}
15265
15266 ins_encode(aarch64_enc_br_con(cmp, lbl));
15267
15268 ins_pipe(pipe_branch);
15269 %}
15270
15271 // counted loop end branch far
15272 // TODO: fixme
15273
15274 // ============================================================================
15275 // inlined locking and unlocking
15276
15277 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15278 %{
15279 match(Set cr (FastLock object box));
15280 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15281
15282 ins_cost(5 * INSN_COST);
15283 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15284
15285 ins_encode %{
15286 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15287 %}
15288
15289 ins_pipe(pipe_serial);
15290 %}
15291
15292 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15293 %{
15294 match(Set cr (FastUnlock object box));
15295 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15296
15297 ins_cost(5 * INSN_COST);
15298 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15299
15300 ins_encode %{
15301 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15302 %}
15303
15304 ins_pipe(pipe_serial);
15305 %}
15306
15307 // ============================================================================
15308 // Safepoint Instructions
15309
15310 // TODO
15311 // provide a near and far version of this code
15312
15313 instruct safePoint(rFlagsReg cr, iRegP poll)
15314 %{
15315 match(SafePoint poll);
15316 effect(KILL cr);
15317
15318 format %{
15319 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15320 %}
15321 ins_encode %{
15322 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15323 %}
15324 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15325 %}
15326
15327
15328 // ============================================================================
15329 // Procedure Call/Return Instructions
15330
15331 // Call Java Static Instruction
15332
15333 instruct CallStaticJavaDirect(method meth)
15334 %{
15335 match(CallStaticJava);
15336
15337 effect(USE meth);
15338
15339 ins_cost(CALL_COST);
15340
15341 format %{ "call,static $meth \t// ==> " %}
15342
15343 ins_encode(aarch64_enc_java_static_call(meth),
15344 aarch64_enc_call_epilog);
15345
15346 ins_pipe(pipe_class_call);
15347 %}
15348
15349 // TO HERE
15350
15351 // Call Java Dynamic Instruction
15352 instruct CallDynamicJavaDirect(method meth)
15353 %{
15354 match(CallDynamicJava);
15355
15356 effect(USE meth);
15357
15358 ins_cost(CALL_COST);
15359
15360 format %{ "CALL,dynamic $meth \t// ==> " %}
15361
15362 ins_encode(aarch64_enc_java_dynamic_call(meth),
15363 aarch64_enc_call_epilog);
15364
15365 ins_pipe(pipe_class_call);
15366 %}
15367
15368 // Call Runtime Instruction
15369
15370 instruct CallRuntimeDirect(method meth)
15371 %{
15372 match(CallRuntime);
15373
15374 effect(USE meth);
15375
15376 ins_cost(CALL_COST);
15377
15378 format %{ "CALL, runtime $meth" %}
15379
15380 ins_encode( aarch64_enc_java_to_runtime(meth) );
15381
15382 ins_pipe(pipe_class_call);
15383 %}
15384
15385 // Call Runtime Instruction
15386
15387 instruct CallLeafDirect(method meth)
15388 %{
15389 match(CallLeaf);
15390
15391 effect(USE meth);
15392
15393 ins_cost(CALL_COST);
15394
15395 format %{ "CALL, runtime leaf $meth" %}
15396
15397 ins_encode( aarch64_enc_java_to_runtime(meth) );
15398
15399 ins_pipe(pipe_class_call);
15400 %}
15401
15402 // Call Runtime Instruction without safepoint and with vector arguments
15403 instruct CallLeafDirectVector(method meth)
15404 %{
15405 match(CallLeafVector);
15406
15407 effect(USE meth);
15408
15409 ins_cost(CALL_COST);
15410
15411 format %{ "CALL, runtime leaf vector $meth" %}
15412
15413 ins_encode(aarch64_enc_java_to_runtime(meth));
15414
15415 ins_pipe(pipe_class_call);
15416 %}
15417
15418 // Call Runtime Instruction
15419
15420 instruct CallLeafNoFPDirect(method meth)
15421 %{
15422 match(CallLeafNoFP);
15423
15424 effect(USE meth);
15425
15426 ins_cost(CALL_COST);
15427
15428 format %{ "CALL, runtime leaf nofp $meth" %}
15429
15430 ins_encode( aarch64_enc_java_to_runtime(meth) );
15431
15432 ins_pipe(pipe_class_call);
15433 %}
15434
15435 // Tail Call; Jump from runtime stub to Java code.
15436 // Also known as an 'interprocedural jump'.
15437 // Target of jump will eventually return to caller.
15438 // TailJump below removes the return address.
15439 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15440 // emitted just above the TailCall which has reset rfp to the caller state.
15441 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15442 %{
15443 match(TailCall jump_target method_ptr);
15444
15445 ins_cost(CALL_COST);
15446
15447 format %{ "br $jump_target\t# $method_ptr holds method" %}
15448
15449 ins_encode(aarch64_enc_tail_call(jump_target));
15450
15451 ins_pipe(pipe_class_call);
15452 %}
15453
15454 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15455 %{
15456 match(TailJump jump_target ex_oop);
15457
15458 ins_cost(CALL_COST);
15459
15460 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15461
15462 ins_encode(aarch64_enc_tail_jmp(jump_target));
15463
15464 ins_pipe(pipe_class_call);
15465 %}
15466
15467 // Forward exception.
15468 instruct ForwardExceptionjmp()
15469 %{
15470 match(ForwardException);
15471 ins_cost(CALL_COST);
15472
15473 format %{ "b forward_exception_stub" %}
15474 ins_encode %{
15475 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15476 %}
15477 ins_pipe(pipe_class_call);
15478 %}
15479
15480 // Create exception oop: created by stack-crawling runtime code.
15481 // Created exception is now available to this handler, and is setup
15482 // just prior to jumping to this handler. No code emitted.
15483 // TODO check
15484 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15485 instruct CreateException(iRegP_R0 ex_oop)
15486 %{
15487 match(Set ex_oop (CreateEx));
15488
15489 format %{ " -- \t// exception oop; no code emitted" %}
15490
15491 size(0);
15492
15493 ins_encode( /*empty*/ );
15494
15495 ins_pipe(pipe_class_empty);
15496 %}
15497
15498 // Rethrow exception: The exception oop will come in the first
15499 // argument position. Then JUMP (not call) to the rethrow stub code.
15500 instruct RethrowException() %{
15501 match(Rethrow);
15502 ins_cost(CALL_COST);
15503
15504 format %{ "b rethrow_stub" %}
15505
15506 ins_encode( aarch64_enc_rethrow() );
15507
15508 ins_pipe(pipe_class_call);
15509 %}
15510
15511
15512 // Return Instruction
15513 // epilog node loads ret address into lr as part of frame pop
15514 instruct Ret()
15515 %{
15516 match(Return);
15517
15518 format %{ "ret\t// return register" %}
15519
15520 ins_encode( aarch64_enc_ret() );
15521
15522 ins_pipe(pipe_branch);
15523 %}
15524
15525 // Die now.
15526 instruct ShouldNotReachHere() %{
15527 match(Halt);
15528
15529 ins_cost(CALL_COST);
15530 format %{ "ShouldNotReachHere" %}
15531
15532 ins_encode %{
15533 if (is_reachable()) {
15534 const char* str = __ code_string(_halt_reason);
15535 __ stop(str);
15536 }
15537 %}
15538
15539 ins_pipe(pipe_class_default);
15540 %}
15541
15542 // ============================================================================
15543 // Partial Subtype Check
15544 //
15545 // superklass array for an instance of the superklass. Set a hidden
15546 // internal cache on a hit (cache is checked with exposed code in
15547 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15548 // encoding ALSO sets flags.
15549
15550 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15551 %{
15552 match(Set result (PartialSubtypeCheck sub super));
15553 predicate(!UseSecondarySupersTable);
15554 effect(KILL cr, KILL temp);
15555
15556 ins_cost(20 * INSN_COST); // slightly larger than the next version
15557 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15558
15559 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15560
15561 opcode(0x1); // Force zero of result reg on hit
15562
15563 ins_pipe(pipe_class_memory);
15564 %}
15565
15566 // Two versions of partialSubtypeCheck, both used when we need to
15567 // search for a super class in the secondary supers array. The first
15568 // is used when we don't know _a priori_ the class being searched
15569 // for. The second, far more common, is used when we do know: this is
15570 // used for instanceof, checkcast, and any case where C2 can determine
15571 // it by constant propagation.
15572
15573 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15574 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15575 rFlagsReg cr)
15576 %{
15577 match(Set result (PartialSubtypeCheck sub super));
15578 predicate(UseSecondarySupersTable);
15579 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15580
15581 ins_cost(10 * INSN_COST); // slightly larger than the next version
15582 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15583
15584 ins_encode %{
15585 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15586 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15587 $vtemp$$FloatRegister,
15588 $result$$Register, /*L_success*/nullptr);
15589 %}
15590
15591 ins_pipe(pipe_class_memory);
15592 %}
15593
15594 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15595 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15596 rFlagsReg cr)
15597 %{
15598 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15599 predicate(UseSecondarySupersTable);
15600 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15601
15602 ins_cost(5 * INSN_COST); // smaller than the next version
15603 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15604
15605 ins_encode %{
15606 bool success = false;
15607 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15608 if (InlineSecondarySupersTest) {
15609 success =
15610 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15611 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15612 $vtemp$$FloatRegister,
15613 $result$$Register,
15614 super_klass_slot);
15615 } else {
15616 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15617 success = (call != nullptr);
15618 }
15619 if (!success) {
15620 ciEnv::current()->record_failure("CodeCache is full");
15621 return;
15622 }
15623 %}
15624
15625 ins_pipe(pipe_class_memory);
15626 %}
15627
15628 // Intrisics for String.compareTo()
15629
15630 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15631 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15632 %{
15633 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15634 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15635 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15636
15637 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15638 ins_encode %{
15639 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15640 __ string_compare($str1$$Register, $str2$$Register,
15641 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15642 $tmp1$$Register, $tmp2$$Register,
15643 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15644 %}
15645 ins_pipe(pipe_class_memory);
15646 %}
15647
15648 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15649 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15650 %{
15651 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15652 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15653 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15654
15655 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15656 ins_encode %{
15657 __ string_compare($str1$$Register, $str2$$Register,
15658 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15659 $tmp1$$Register, $tmp2$$Register,
15660 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15661 %}
15662 ins_pipe(pipe_class_memory);
15663 %}
15664
15665 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15666 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15667 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15668 %{
15669 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15670 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15671 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15672 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15673
15674 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15675 ins_encode %{
15676 __ string_compare($str1$$Register, $str2$$Register,
15677 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15678 $tmp1$$Register, $tmp2$$Register,
15679 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15680 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15681 %}
15682 ins_pipe(pipe_class_memory);
15683 %}
15684
15685 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15686 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15687 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15688 %{
15689 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15690 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15691 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15692 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15693
15694 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15695 ins_encode %{
15696 __ string_compare($str1$$Register, $str2$$Register,
15697 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15698 $tmp1$$Register, $tmp2$$Register,
15699 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15700 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15701 %}
15702 ins_pipe(pipe_class_memory);
15703 %}
15704
15705 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15706 // these string_compare variants as NEON register type for convenience so that the prototype of
15707 // string_compare can be shared with all variants.
15708
15709 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15710 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15711 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15712 pRegGov_P1 pgtmp2, rFlagsReg cr)
15713 %{
15714 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15715 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15716 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15717 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15718
15719 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15720 ins_encode %{
15721 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15722 __ string_compare($str1$$Register, $str2$$Register,
15723 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15724 $tmp1$$Register, $tmp2$$Register,
15725 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15726 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15727 StrIntrinsicNode::LL);
15728 %}
15729 ins_pipe(pipe_class_memory);
15730 %}
15731
15732 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15733 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15734 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15735 pRegGov_P1 pgtmp2, rFlagsReg cr)
15736 %{
15737 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15738 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15739 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15740 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15741
15742 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15743 ins_encode %{
15744 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15745 __ string_compare($str1$$Register, $str2$$Register,
15746 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15747 $tmp1$$Register, $tmp2$$Register,
15748 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15749 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15750 StrIntrinsicNode::LU);
15751 %}
15752 ins_pipe(pipe_class_memory);
15753 %}
15754
15755 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15756 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15757 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15758 pRegGov_P1 pgtmp2, rFlagsReg cr)
15759 %{
15760 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15761 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15762 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15763 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15764
15765 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15766 ins_encode %{
15767 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15768 __ string_compare($str1$$Register, $str2$$Register,
15769 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15770 $tmp1$$Register, $tmp2$$Register,
15771 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15772 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15773 StrIntrinsicNode::UL);
15774 %}
15775 ins_pipe(pipe_class_memory);
15776 %}
15777
15778 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15779 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15780 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15781 pRegGov_P1 pgtmp2, rFlagsReg cr)
15782 %{
15783 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15784 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15785 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15786 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15787
15788 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15789 ins_encode %{
15790 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15791 __ string_compare($str1$$Register, $str2$$Register,
15792 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15793 $tmp1$$Register, $tmp2$$Register,
15794 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15795 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15796 StrIntrinsicNode::UU);
15797 %}
15798 ins_pipe(pipe_class_memory);
15799 %}
15800
15801 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15802 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15803 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15804 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15805 %{
15806 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15807 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15808 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15809 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15810 TEMP vtmp0, TEMP vtmp1, KILL cr);
15811 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15812 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15813
15814 ins_encode %{
15815 __ string_indexof($str1$$Register, $str2$$Register,
15816 $cnt1$$Register, $cnt2$$Register,
15817 $tmp1$$Register, $tmp2$$Register,
15818 $tmp3$$Register, $tmp4$$Register,
15819 $tmp5$$Register, $tmp6$$Register,
15820 -1, $result$$Register, StrIntrinsicNode::UU);
15821 %}
15822 ins_pipe(pipe_class_memory);
15823 %}
15824
15825 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15826 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15827 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15828 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15829 %{
15830 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15831 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15832 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15833 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15834 TEMP vtmp0, TEMP vtmp1, KILL cr);
15835 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15836 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15837
15838 ins_encode %{
15839 __ string_indexof($str1$$Register, $str2$$Register,
15840 $cnt1$$Register, $cnt2$$Register,
15841 $tmp1$$Register, $tmp2$$Register,
15842 $tmp3$$Register, $tmp4$$Register,
15843 $tmp5$$Register, $tmp6$$Register,
15844 -1, $result$$Register, StrIntrinsicNode::LL);
15845 %}
15846 ins_pipe(pipe_class_memory);
15847 %}
15848
15849 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15850 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15851 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15852 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15853 %{
15854 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15855 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15856 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15857 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15858 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15859 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15860 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15861
15862 ins_encode %{
15863 __ string_indexof($str1$$Register, $str2$$Register,
15864 $cnt1$$Register, $cnt2$$Register,
15865 $tmp1$$Register, $tmp2$$Register,
15866 $tmp3$$Register, $tmp4$$Register,
15867 $tmp5$$Register, $tmp6$$Register,
15868 -1, $result$$Register, StrIntrinsicNode::UL);
15869 %}
15870 ins_pipe(pipe_class_memory);
15871 %}
15872
15873 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15874 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15875 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15876 %{
15877 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15878 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15879 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15880 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15881 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15882 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15883
15884 ins_encode %{
15885 int icnt2 = (int)$int_cnt2$$constant;
15886 __ string_indexof($str1$$Register, $str2$$Register,
15887 $cnt1$$Register, zr,
15888 $tmp1$$Register, $tmp2$$Register,
15889 $tmp3$$Register, $tmp4$$Register, zr, zr,
15890 icnt2, $result$$Register, StrIntrinsicNode::UU);
15891 %}
15892 ins_pipe(pipe_class_memory);
15893 %}
15894
15895 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15896 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15897 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15898 %{
15899 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15900 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15901 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15902 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15903 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15904 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15905
15906 ins_encode %{
15907 int icnt2 = (int)$int_cnt2$$constant;
15908 __ string_indexof($str1$$Register, $str2$$Register,
15909 $cnt1$$Register, zr,
15910 $tmp1$$Register, $tmp2$$Register,
15911 $tmp3$$Register, $tmp4$$Register, zr, zr,
15912 icnt2, $result$$Register, StrIntrinsicNode::LL);
15913 %}
15914 ins_pipe(pipe_class_memory);
15915 %}
15916
15917 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15918 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15919 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15920 %{
15921 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15922 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15923 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15924 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15925 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15926 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15927
15928 ins_encode %{
15929 int icnt2 = (int)$int_cnt2$$constant;
15930 __ string_indexof($str1$$Register, $str2$$Register,
15931 $cnt1$$Register, zr,
15932 $tmp1$$Register, $tmp2$$Register,
15933 $tmp3$$Register, $tmp4$$Register, zr, zr,
15934 icnt2, $result$$Register, StrIntrinsicNode::UL);
15935 %}
15936 ins_pipe(pipe_class_memory);
15937 %}
15938
15939 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15940 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15941 iRegINoSp tmp3, rFlagsReg cr)
15942 %{
15943 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15944 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15945 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15946 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15947
15948 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15949
15950 ins_encode %{
15951 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15952 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15953 $tmp3$$Register);
15954 %}
15955 ins_pipe(pipe_class_memory);
15956 %}
15957
15958 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15959 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15960 iRegINoSp tmp3, rFlagsReg cr)
15961 %{
15962 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15963 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15964 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15965 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15966
15967 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15968
15969 ins_encode %{
15970 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15971 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15972 $tmp3$$Register);
15973 %}
15974 ins_pipe(pipe_class_memory);
15975 %}
15976
15977 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15978 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15979 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15980 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15981 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15982 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15983 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15984 ins_encode %{
15985 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15986 $result$$Register, $ztmp1$$FloatRegister,
15987 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15988 $ptmp$$PRegister, true /* isL */);
15989 %}
15990 ins_pipe(pipe_class_memory);
15991 %}
15992
15993 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15994 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15995 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15996 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15997 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15998 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15999 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16000 ins_encode %{
16001 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16002 $result$$Register, $ztmp1$$FloatRegister,
16003 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16004 $ptmp$$PRegister, false /* isL */);
16005 %}
16006 ins_pipe(pipe_class_memory);
16007 %}
16008
16009 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16010 iRegI_R0 result, rFlagsReg cr)
16011 %{
16012 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16013 match(Set result (StrEquals (Binary str1 str2) cnt));
16014 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16015
16016 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16017 ins_encode %{
16018 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16019 __ string_equals($str1$$Register, $str2$$Register,
16020 $result$$Register, $cnt$$Register);
16021 %}
16022 ins_pipe(pipe_class_memory);
16023 %}
16024
16025 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16026 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16027 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16028 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16029 iRegP_R10 tmp, rFlagsReg cr)
16030 %{
16031 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16032 match(Set result (AryEq ary1 ary2));
16033 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16034 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16035 TEMP vtmp6, TEMP vtmp7, KILL cr);
16036
16037 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16038 ins_encode %{
16039 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16040 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16041 $result$$Register, $tmp$$Register, 1);
16042 if (tpc == nullptr) {
16043 ciEnv::current()->record_failure("CodeCache is full");
16044 return;
16045 }
16046 %}
16047 ins_pipe(pipe_class_memory);
16048 %}
16049
16050 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16051 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16052 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16053 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16054 iRegP_R10 tmp, rFlagsReg cr)
16055 %{
16056 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16057 match(Set result (AryEq ary1 ary2));
16058 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16059 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16060 TEMP vtmp6, TEMP vtmp7, KILL cr);
16061
16062 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16063 ins_encode %{
16064 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16065 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16066 $result$$Register, $tmp$$Register, 2);
16067 if (tpc == nullptr) {
16068 ciEnv::current()->record_failure("CodeCache is full");
16069 return;
16070 }
16071 %}
16072 ins_pipe(pipe_class_memory);
16073 %}
16074
16075 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16076 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16077 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16078 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16079 %{
16080 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16081 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16082 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16083
16084 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16085 ins_encode %{
16086 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16087 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16088 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16089 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16090 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16091 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16092 (BasicType)$basic_type$$constant);
16093 if (tpc == nullptr) {
16094 ciEnv::current()->record_failure("CodeCache is full");
16095 return;
16096 }
16097 %}
16098 ins_pipe(pipe_class_memory);
16099 %}
16100
16101 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16102 %{
16103 match(Set result (CountPositives ary1 len));
16104 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16105 format %{ "count positives byte[] $ary1,$len -> $result" %}
16106 ins_encode %{
16107 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16108 if (tpc == nullptr) {
16109 ciEnv::current()->record_failure("CodeCache is full");
16110 return;
16111 }
16112 %}
16113 ins_pipe( pipe_slow );
16114 %}
16115
16116 // fast char[] to byte[] compression
16117 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16118 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16119 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16120 iRegI_R0 result, rFlagsReg cr)
16121 %{
16122 match(Set result (StrCompressedCopy src (Binary dst len)));
16123 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16124 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16125
16126 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16127 ins_encode %{
16128 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16129 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16130 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16131 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16132 %}
16133 ins_pipe(pipe_slow);
16134 %}
16135
16136 // fast byte[] to char[] inflation
16137 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16138 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16139 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16140 %{
16141 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16142 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16143 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16144 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16145
16146 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16147 ins_encode %{
16148 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16149 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16150 $vtmp2$$FloatRegister, $tmp$$Register);
16151 if (tpc == nullptr) {
16152 ciEnv::current()->record_failure("CodeCache is full");
16153 return;
16154 }
16155 %}
16156 ins_pipe(pipe_class_memory);
16157 %}
16158
16159 // encode char[] to byte[] in ISO_8859_1
16160 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16161 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16162 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16163 iRegI_R0 result, rFlagsReg cr)
16164 %{
16165 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16166 match(Set result (EncodeISOArray src (Binary dst len)));
16167 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16168 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16169
16170 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16171 ins_encode %{
16172 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16173 $result$$Register, false,
16174 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16175 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16176 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16177 %}
16178 ins_pipe(pipe_class_memory);
16179 %}
16180
16181 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16182 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16183 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16184 iRegI_R0 result, rFlagsReg cr)
16185 %{
16186 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16187 match(Set result (EncodeISOArray src (Binary dst len)));
16188 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16189 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16190
16191 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16192 ins_encode %{
16193 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16194 $result$$Register, true,
16195 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16196 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16197 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16198 %}
16199 ins_pipe(pipe_class_memory);
16200 %}
16201
16202 //----------------------------- CompressBits/ExpandBits ------------------------
16203
16204 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16205 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16206 match(Set dst (CompressBits src mask));
16207 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16208 format %{ "mov $tsrc, $src\n\t"
16209 "mov $tmask, $mask\n\t"
16210 "bext $tdst, $tsrc, $tmask\n\t"
16211 "mov $dst, $tdst"
16212 %}
16213 ins_encode %{
16214 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16215 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16216 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16217 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16218 %}
16219 ins_pipe(pipe_slow);
16220 %}
16221
16222 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16223 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16224 match(Set dst (CompressBits (LoadI mem) mask));
16225 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16226 format %{ "ldrs $tsrc, $mem\n\t"
16227 "ldrs $tmask, $mask\n\t"
16228 "bext $tdst, $tsrc, $tmask\n\t"
16229 "mov $dst, $tdst"
16230 %}
16231 ins_encode %{
16232 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16233 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16234 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16235 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16236 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16237 %}
16238 ins_pipe(pipe_slow);
16239 %}
16240
16241 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16242 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16243 match(Set dst (CompressBits src mask));
16244 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16245 format %{ "mov $tsrc, $src\n\t"
16246 "mov $tmask, $mask\n\t"
16247 "bext $tdst, $tsrc, $tmask\n\t"
16248 "mov $dst, $tdst"
16249 %}
16250 ins_encode %{
16251 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16252 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16253 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16254 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16255 %}
16256 ins_pipe(pipe_slow);
16257 %}
16258
16259 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16260 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16261 match(Set dst (CompressBits (LoadL mem) mask));
16262 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16263 format %{ "ldrd $tsrc, $mem\n\t"
16264 "ldrd $tmask, $mask\n\t"
16265 "bext $tdst, $tsrc, $tmask\n\t"
16266 "mov $dst, $tdst"
16267 %}
16268 ins_encode %{
16269 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16270 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16271 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16272 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16273 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16274 %}
16275 ins_pipe(pipe_slow);
16276 %}
16277
16278 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16279 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16280 match(Set dst (ExpandBits src mask));
16281 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16282 format %{ "mov $tsrc, $src\n\t"
16283 "mov $tmask, $mask\n\t"
16284 "bdep $tdst, $tsrc, $tmask\n\t"
16285 "mov $dst, $tdst"
16286 %}
16287 ins_encode %{
16288 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16289 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16290 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16291 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16292 %}
16293 ins_pipe(pipe_slow);
16294 %}
16295
16296 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16297 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16298 match(Set dst (ExpandBits (LoadI mem) mask));
16299 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16300 format %{ "ldrs $tsrc, $mem\n\t"
16301 "ldrs $tmask, $mask\n\t"
16302 "bdep $tdst, $tsrc, $tmask\n\t"
16303 "mov $dst, $tdst"
16304 %}
16305 ins_encode %{
16306 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16307 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16308 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16309 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16310 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16311 %}
16312 ins_pipe(pipe_slow);
16313 %}
16314
16315 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16316 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16317 match(Set dst (ExpandBits src mask));
16318 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16319 format %{ "mov $tsrc, $src\n\t"
16320 "mov $tmask, $mask\n\t"
16321 "bdep $tdst, $tsrc, $tmask\n\t"
16322 "mov $dst, $tdst"
16323 %}
16324 ins_encode %{
16325 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16326 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16327 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16328 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16329 %}
16330 ins_pipe(pipe_slow);
16331 %}
16332
16333
16334 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16335 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16336 match(Set dst (ExpandBits (LoadL mem) mask));
16337 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16338 format %{ "ldrd $tsrc, $mem\n\t"
16339 "ldrd $tmask, $mask\n\t"
16340 "bdep $tdst, $tsrc, $tmask\n\t"
16341 "mov $dst, $tdst"
16342 %}
16343 ins_encode %{
16344 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16345 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16346 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16347 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16348 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16349 %}
16350 ins_pipe(pipe_slow);
16351 %}
16352
16353 //----------------------------- Reinterpret ----------------------------------
16354 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16355 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16356 match(Set dst (ReinterpretHF2S src));
16357 format %{ "reinterpretHF2S $dst, $src" %}
16358 ins_encode %{
16359 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16360 %}
16361 ins_pipe(pipe_slow);
16362 %}
16363
16364 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16365 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16366 match(Set dst (ReinterpretS2HF src));
16367 format %{ "reinterpretS2HF $dst, $src" %}
16368 ins_encode %{
16369 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16370 %}
16371 ins_pipe(pipe_slow);
16372 %}
16373
16374 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16375 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16376 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16377 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16378 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16379 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16380 // can be omitted in this pattern, resulting in -
16381 // fcvt $dst, $src // Convert float to half-precision float
16382 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16383 %{
16384 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16385 format %{ "convF2HFAndS2HF $dst, $src" %}
16386 ins_encode %{
16387 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16388 %}
16389 ins_pipe(pipe_slow);
16390 %}
16391
16392 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16393 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16394 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16395 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16396 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16397 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16398 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16399 // resulting in -
16400 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16401 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16402 %{
16403 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16404 format %{ "convHF2SAndHF2F $dst, $src" %}
16405 ins_encode %{
16406 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16407 %}
16408 ins_pipe(pipe_slow);
16409 %}
16410
16411 // ============================================================================
16412 // This name is KNOWN by the ADLC and cannot be changed.
16413 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16414 // for this guy.
16415 instruct tlsLoadP(thread_RegP dst)
16416 %{
16417 match(Set dst (ThreadLocal));
16418
16419 ins_cost(0);
16420
16421 format %{ " -- \t// $dst=Thread::current(), empty" %}
16422
16423 size(0);
16424
16425 ins_encode( /*empty*/ );
16426
16427 ins_pipe(pipe_class_empty);
16428 %}
16429
16430 //----------PEEPHOLE RULES-----------------------------------------------------
16431 // These must follow all instruction definitions as they use the names
16432 // defined in the instructions definitions.
16433 //
16434 // peepmatch ( root_instr_name [preceding_instruction]* );
16435 //
16436 // peepconstraint %{
16437 // (instruction_number.operand_name relational_op instruction_number.operand_name
16438 // [, ...] );
16439 // // instruction numbers are zero-based using left to right order in peepmatch
16440 //
16441 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16442 // // provide an instruction_number.operand_name for each operand that appears
16443 // // in the replacement instruction's match rule
16444 //
16445 // ---------VM FLAGS---------------------------------------------------------
16446 //
16447 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16448 //
16449 // Each peephole rule is given an identifying number starting with zero and
16450 // increasing by one in the order seen by the parser. An individual peephole
16451 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16452 // on the command-line.
16453 //
16454 // ---------CURRENT LIMITATIONS----------------------------------------------
16455 //
16456 // Only match adjacent instructions in same basic block
16457 // Only equality constraints
16458 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16459 // Only one replacement instruction
16460 //
16461 // ---------EXAMPLE----------------------------------------------------------
16462 //
16463 // // pertinent parts of existing instructions in architecture description
16464 // instruct movI(iRegINoSp dst, iRegI src)
16465 // %{
16466 // match(Set dst (CopyI src));
16467 // %}
16468 //
16469 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16470 // %{
16471 // match(Set dst (AddI dst src));
16472 // effect(KILL cr);
16473 // %}
16474 //
16475 // // Change (inc mov) to lea
16476 // peephole %{
16477 // // increment preceded by register-register move
16478 // peepmatch ( incI_iReg movI );
16479 // // require that the destination register of the increment
16480 // // match the destination register of the move
16481 // peepconstraint ( 0.dst == 1.dst );
16482 // // construct a replacement instruction that sets
16483 // // the destination to ( move's source register + one )
16484 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16485 // %}
16486 //
16487
16488 // Implementation no longer uses movX instructions since
16489 // machine-independent system no longer uses CopyX nodes.
16490 //
16491 // peephole
16492 // %{
16493 // peepmatch (incI_iReg movI);
16494 // peepconstraint (0.dst == 1.dst);
16495 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16496 // %}
16497
16498 // peephole
16499 // %{
16500 // peepmatch (decI_iReg movI);
16501 // peepconstraint (0.dst == 1.dst);
16502 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16503 // %}
16504
16505 // peephole
16506 // %{
16507 // peepmatch (addI_iReg_imm movI);
16508 // peepconstraint (0.dst == 1.dst);
16509 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16510 // %}
16511
16512 // peephole
16513 // %{
16514 // peepmatch (incL_iReg movL);
16515 // peepconstraint (0.dst == 1.dst);
16516 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16517 // %}
16518
16519 // peephole
16520 // %{
16521 // peepmatch (decL_iReg movL);
16522 // peepconstraint (0.dst == 1.dst);
16523 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16524 // %}
16525
16526 // peephole
16527 // %{
16528 // peepmatch (addL_iReg_imm movL);
16529 // peepconstraint (0.dst == 1.dst);
16530 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16531 // %}
16532
16533 // peephole
16534 // %{
16535 // peepmatch (addP_iReg_imm movP);
16536 // peepconstraint (0.dst == 1.dst);
16537 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16538 // %}
16539
16540 // // Change load of spilled value to only a spill
16541 // instruct storeI(memory mem, iRegI src)
16542 // %{
16543 // match(Set mem (StoreI mem src));
16544 // %}
16545 //
16546 // instruct loadI(iRegINoSp dst, memory mem)
16547 // %{
16548 // match(Set dst (LoadI mem));
16549 // %}
16550 //
16551
16552 //----------SMARTSPILL RULES---------------------------------------------------
16553 // These must follow all instruction definitions as they use the names
16554 // defined in the instructions definitions.
16555
16556 // Local Variables:
16557 // mode: c++
16558 // End: