1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 if (UseCompressedClassPointers) {
2237 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmpw rscratch1, r10");
2240 } else {
2241 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2242 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2243 st->print_cr("\tcmp rscratch1, r10");
2244 }
2245 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2246 }
2247 #endif
2248
2249 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2250 {
2251 __ ic_check(InteriorEntryAlignment);
2252 }
2253
2254 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2255 {
2256 return MachNode::size(ra_);
2257 }
2258
2259 // REQUIRED EMIT CODE
2260
2261 //=============================================================================
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 address base = __ start_a_stub(size_deopt_handler());
2269 if (base == nullptr) {
2270 ciEnv::current()->record_failure("CodeCache is full");
2271 return 0; // CodeBuffer::expand failed
2272 }
2273
2274 int offset = __ offset();
2275 Label start;
2276 __ bind(start);
2277 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 int entry_offset = __ offset();
2280 __ b(start);
2281
2282 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2283 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2284 "out of bounds read in post-call NOP check");
2285 __ end_a_stub();
2286 return entry_offset;
2287 }
2288
2289 // REQUIRED MATCHER CODE
2290
2291 //=============================================================================
2292
2293 bool Matcher::match_rule_supported(int opcode) {
2294 if (!has_match_rule(opcode))
2295 return false;
2296
2297 switch (opcode) {
2298 case Op_OnSpinWait:
2299 return VM_Version::supports_on_spin_wait();
2300 case Op_CacheWB:
2301 case Op_CacheWBPreSync:
2302 case Op_CacheWBPostSync:
2303 if (!VM_Version::supports_data_cache_line_flush()) {
2304 return false;
2305 }
2306 break;
2307 case Op_ExpandBits:
2308 case Op_CompressBits:
2309 if (!VM_Version::supports_svebitperm()) {
2310 return false;
2311 }
2312 break;
2313 case Op_FmaF:
2314 case Op_FmaD:
2315 case Op_FmaVF:
2316 case Op_FmaVD:
2317 if (!UseFMA) {
2318 return false;
2319 }
2320 break;
2321 case Op_FmaHF:
2322 // UseFMA flag also needs to be checked along with FEAT_FP16
2323 if (!UseFMA || !is_feat_fp16_supported()) {
2324 return false;
2325 }
2326 break;
2327 case Op_AddHF:
2328 case Op_SubHF:
2329 case Op_MulHF:
2330 case Op_DivHF:
2331 case Op_MinHF:
2332 case Op_MaxHF:
2333 case Op_SqrtHF:
2334 // Half-precision floating point scalar operations require FEAT_FP16
2335 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2336 // features are supported.
2337 if (!is_feat_fp16_supported()) {
2338 return false;
2339 }
2340 break;
2341 }
2342
2343 return true; // Per default match rules are supported.
2344 }
2345
2346 const RegMask* Matcher::predicate_reg_mask(void) {
2347 return &_PR_REG_mask;
2348 }
2349
2350 bool Matcher::supports_vector_calling_convention(void) {
2351 return EnableVectorSupport;
2352 }
2353
2354 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2355 assert(EnableVectorSupport, "sanity");
2356 int lo = V0_num;
2357 int hi = V0_H_num;
2358 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2359 hi = V0_K_num;
2360 }
2361 return OptoRegPair(hi, lo);
2362 }
2363
2364 // Is this branch offset short enough that a short branch can be used?
2365 //
2366 // NOTE: If the platform does not provide any short branch variants, then
2367 // this method should return false for offset 0.
2368 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2369 // The passed offset is relative to address of the branch.
2370
2371 return (-32768 <= offset && offset < 32768);
2372 }
2373
2374 // Vector width in bytes.
2375 int Matcher::vector_width_in_bytes(BasicType bt) {
2376 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2377 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2378 // Minimum 2 values in vector
2379 if (size < 2*type2aelembytes(bt)) size = 0;
2380 // But never < 4
2381 if (size < 4) size = 0;
2382 return size;
2383 }
2384
2385 // Limits on vector size (number of elements) loaded into vector.
2386 int Matcher::max_vector_size(const BasicType bt) {
2387 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2388 }
2389
2390 int Matcher::min_vector_size(const BasicType bt) {
2391 // Usually, the shortest vector length supported by AArch64 ISA and
2392 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2393 // vectors in a few special cases.
2394 int size;
2395 switch(bt) {
2396 case T_BOOLEAN:
2397 // Load/store a vector mask with only 2 elements for vector types
2398 // such as "2I/2F/2L/2D".
2399 size = 2;
2400 break;
2401 case T_BYTE:
2402 // Generate a "4B" vector, to support vector cast between "8B/16B"
2403 // and "4S/4I/4L/4F/4D".
2404 size = 4;
2405 break;
2406 case T_SHORT:
2407 // Generate a "2S" vector, to support vector cast between "4S/8S"
2408 // and "2I/2L/2F/2D".
2409 size = 2;
2410 break;
2411 default:
2412 // Limit the min vector length to 64-bit.
2413 size = 8 / type2aelembytes(bt);
2414 // The number of elements in a vector should be at least 2.
2415 size = MAX2(size, 2);
2416 }
2417
2418 int max_size = max_vector_size(bt);
2419 return MIN2(size, max_size);
2420 }
2421
2422 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2423 return Matcher::max_vector_size(bt);
2424 }
2425
2426 // Actual max scalable vector register length.
2427 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2428 return Matcher::max_vector_size(bt);
2429 }
2430
2431 // Vector ideal reg.
2432 uint Matcher::vector_ideal_reg(int len) {
2433 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2434 return Op_VecA;
2435 }
2436 switch(len) {
2437 // For 16-bit/32-bit mask vector, reuse VecD.
2438 case 2:
2439 case 4:
2440 case 8: return Op_VecD;
2441 case 16: return Op_VecX;
2442 }
2443 ShouldNotReachHere();
2444 return 0;
2445 }
2446
2447 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2448 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2449 switch (ideal_reg) {
2450 case Op_VecA: return new vecAOper();
2451 case Op_VecD: return new vecDOper();
2452 case Op_VecX: return new vecXOper();
2453 }
2454 ShouldNotReachHere();
2455 return nullptr;
2456 }
2457
2458 bool Matcher::is_reg2reg_move(MachNode* m) {
2459 return false;
2460 }
2461
2462 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2463 return false;
2464 }
2465
2466 bool Matcher::is_generic_vector(MachOper* opnd) {
2467 return opnd->opcode() == VREG;
2468 }
2469
2470 // Return whether or not this register is ever used as an argument.
2471 // This function is used on startup to build the trampoline stubs in
2472 // generateOptoStub. Registers not mentioned will be killed by the VM
2473 // call in the trampoline, and arguments in those registers not be
2474 // available to the callee.
2475 bool Matcher::can_be_java_arg(int reg)
2476 {
2477 return
2478 reg == R0_num || reg == R0_H_num ||
2479 reg == R1_num || reg == R1_H_num ||
2480 reg == R2_num || reg == R2_H_num ||
2481 reg == R3_num || reg == R3_H_num ||
2482 reg == R4_num || reg == R4_H_num ||
2483 reg == R5_num || reg == R5_H_num ||
2484 reg == R6_num || reg == R6_H_num ||
2485 reg == R7_num || reg == R7_H_num ||
2486 reg == V0_num || reg == V0_H_num ||
2487 reg == V1_num || reg == V1_H_num ||
2488 reg == V2_num || reg == V2_H_num ||
2489 reg == V3_num || reg == V3_H_num ||
2490 reg == V4_num || reg == V4_H_num ||
2491 reg == V5_num || reg == V5_H_num ||
2492 reg == V6_num || reg == V6_H_num ||
2493 reg == V7_num || reg == V7_H_num;
2494 }
2495
2496 bool Matcher::is_spillable_arg(int reg)
2497 {
2498 return can_be_java_arg(reg);
2499 }
2500
2501 uint Matcher::int_pressure_limit()
2502 {
2503 // JDK-8183543: When taking the number of available registers as int
2504 // register pressure threshold, the jtreg test:
2505 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2506 // failed due to C2 compilation failure with
2507 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2508 //
2509 // A derived pointer is live at CallNode and then is flagged by RA
2510 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2511 // derived pointers and lastly fail to spill after reaching maximum
2512 // number of iterations. Lowering the default pressure threshold to
2513 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2514 // a high register pressure area of the code so that split_DEF can
2515 // generate DefinitionSpillCopy for the derived pointer.
2516 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2517 if (!PreserveFramePointer) {
2518 // When PreserveFramePointer is off, frame pointer is allocatable,
2519 // but different from other SOC registers, it is excluded from
2520 // fatproj's mask because its save type is No-Save. Decrease 1 to
2521 // ensure high pressure at fatproj when PreserveFramePointer is off.
2522 // See check_pressure_at_fatproj().
2523 default_int_pressure_threshold--;
2524 }
2525 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2526 }
2527
2528 uint Matcher::float_pressure_limit()
2529 {
2530 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2531 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2532 }
2533
2534 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2535 return false;
2536 }
2537
2538 const RegMask& Matcher::divI_proj_mask() {
2539 ShouldNotReachHere();
2540 return RegMask::EMPTY;
2541 }
2542
2543 // Register for MODI projection of divmodI.
2544 const RegMask& Matcher::modI_proj_mask() {
2545 ShouldNotReachHere();
2546 return RegMask::EMPTY;
2547 }
2548
2549 // Register for DIVL projection of divmodL.
2550 const RegMask& Matcher::divL_proj_mask() {
2551 ShouldNotReachHere();
2552 return RegMask::EMPTY;
2553 }
2554
2555 // Register for MODL projection of divmodL.
2556 const RegMask& Matcher::modL_proj_mask() {
2557 ShouldNotReachHere();
2558 return RegMask::EMPTY;
2559 }
2560
2561 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2562 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2563 Node* u = addp->fast_out(i);
2564 if (u->is_LoadStore()) {
2565 // On AArch64, LoadStoreNodes (i.e. compare and swap
2566 // instructions) only take register indirect as an operand, so
2567 // any attempt to use an AddPNode as an input to a LoadStoreNode
2568 // must fail.
2569 return false;
2570 }
2571 if (u->is_Mem()) {
2572 int opsize = u->as_Mem()->memory_size();
2573 assert(opsize > 0, "unexpected memory operand size");
2574 if (u->as_Mem()->memory_size() != (1<<shift)) {
2575 return false;
2576 }
2577 }
2578 }
2579 return true;
2580 }
2581
2582 // Convert BoolTest condition to Assembler condition.
2583 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2584 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2585 Assembler::Condition result;
2586 switch(cond) {
2587 case BoolTest::eq:
2588 result = Assembler::EQ; break;
2589 case BoolTest::ne:
2590 result = Assembler::NE; break;
2591 case BoolTest::le:
2592 result = Assembler::LE; break;
2593 case BoolTest::ge:
2594 result = Assembler::GE; break;
2595 case BoolTest::lt:
2596 result = Assembler::LT; break;
2597 case BoolTest::gt:
2598 result = Assembler::GT; break;
2599 case BoolTest::ule:
2600 result = Assembler::LS; break;
2601 case BoolTest::uge:
2602 result = Assembler::HS; break;
2603 case BoolTest::ult:
2604 result = Assembler::LO; break;
2605 case BoolTest::ugt:
2606 result = Assembler::HI; break;
2607 case BoolTest::overflow:
2608 result = Assembler::VS; break;
2609 case BoolTest::no_overflow:
2610 result = Assembler::VC; break;
2611 default:
2612 ShouldNotReachHere();
2613 return Assembler::Condition(-1);
2614 }
2615
2616 // Check conversion
2617 if (cond & BoolTest::unsigned_compare) {
2618 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2619 } else {
2620 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2621 }
2622
2623 return result;
2624 }
2625
2626 // Binary src (Replicate con)
2627 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2628 if (n == nullptr || m == nullptr) {
2629 return false;
2630 }
2631
2632 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2633 return false;
2634 }
2635
2636 Node* imm_node = m->in(1);
2637 if (!imm_node->is_Con()) {
2638 return false;
2639 }
2640
2641 const Type* t = imm_node->bottom_type();
2642 if (!(t->isa_int() || t->isa_long())) {
2643 return false;
2644 }
2645
2646 switch (n->Opcode()) {
2647 case Op_AndV:
2648 case Op_OrV:
2649 case Op_XorV: {
2650 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2651 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2652 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2653 }
2654 case Op_AddVB:
2655 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2656 case Op_AddVS:
2657 case Op_AddVI:
2658 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2659 case Op_AddVL:
2660 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2661 default:
2662 return false;
2663 }
2664 }
2665
2666 // (XorV src (Replicate m1))
2667 // (XorVMask src (MaskAll m1))
2668 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2669 if (n != nullptr && m != nullptr) {
2670 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2671 VectorNode::is_all_ones_vector(m);
2672 }
2673 return false;
2674 }
2675
2676 // Should the matcher clone input 'm' of node 'n'?
2677 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2678 if (is_vshift_con_pattern(n, m) ||
2679 is_vector_bitwise_not_pattern(n, m) ||
2680 is_valid_sve_arith_imm_pattern(n, m) ||
2681 is_encode_and_store_pattern(n, m)) {
2682 mstack.push(m, Visit);
2683 return true;
2684 }
2685 return false;
2686 }
2687
2688 // Should the Matcher clone shifts on addressing modes, expecting them
2689 // to be subsumed into complex addressing expressions or compute them
2690 // into registers?
2691 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2692
2693 // Loads and stores with indirect memory input (e.g., volatile loads and
2694 // stores) do not subsume the input into complex addressing expressions. If
2695 // the addressing expression is input to at least one such load or store, do
2696 // not clone the addressing expression. Query needs_acquiring_load and
2697 // needs_releasing_store as a proxy for indirect memory input, as it is not
2698 // possible to directly query for indirect memory input at this stage.
2699 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2700 Node* n = m->fast_out(i);
2701 if (n->is_Load() && needs_acquiring_load(n)) {
2702 return false;
2703 }
2704 if (n->is_Store() && needs_releasing_store(n)) {
2705 return false;
2706 }
2707 }
2708
2709 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2710 return true;
2711 }
2712
2713 Node *off = m->in(AddPNode::Offset);
2714 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2715 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2716 // Are there other uses besides address expressions?
2717 !is_visited(off)) {
2718 address_visited.set(off->_idx); // Flag as address_visited
2719 mstack.push(off->in(2), Visit);
2720 Node *conv = off->in(1);
2721 if (conv->Opcode() == Op_ConvI2L &&
2722 // Are there other uses besides address expressions?
2723 !is_visited(conv)) {
2724 address_visited.set(conv->_idx); // Flag as address_visited
2725 mstack.push(conv->in(1), Pre_Visit);
2726 } else {
2727 mstack.push(conv, Pre_Visit);
2728 }
2729 address_visited.test_set(m->_idx); // Flag as address_visited
2730 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2731 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2732 return true;
2733 } else if (off->Opcode() == Op_ConvI2L &&
2734 // Are there other uses besides address expressions?
2735 !is_visited(off)) {
2736 address_visited.test_set(m->_idx); // Flag as address_visited
2737 address_visited.set(off->_idx); // Flag as address_visited
2738 mstack.push(off->in(1), Pre_Visit);
2739 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2740 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2741 return true;
2742 }
2743 return false;
2744 }
2745
2746 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2747 { \
2748 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2749 guarantee(DISP == 0, "mode not permitted for volatile"); \
2750 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2751 __ INSN(REG, as_Register(BASE)); \
2752 }
2753
2754
2755 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2756 {
2757 Address::extend scale;
2758
2759 // Hooboy, this is fugly. We need a way to communicate to the
2760 // encoder that the index needs to be sign extended, so we have to
2761 // enumerate all the cases.
2762 switch (opcode) {
2763 case INDINDEXSCALEDI2L:
2764 case INDINDEXSCALEDI2LN:
2765 case INDINDEXI2L:
2766 case INDINDEXI2LN:
2767 scale = Address::sxtw(size);
2768 break;
2769 default:
2770 scale = Address::lsl(size);
2771 }
2772
2773 if (index == -1) {
2774 return Address(base, disp);
2775 } else {
2776 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2777 return Address(base, as_Register(index), scale);
2778 }
2779 }
2780
2781
2782 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2783 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2784 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2785 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2786 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2787
2788 // Used for all non-volatile memory accesses. The use of
2789 // $mem->opcode() to discover whether this pattern uses sign-extended
2790 // offsets is something of a kludge.
2791 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2792 Register reg, int opcode,
2793 Register base, int index, int scale, int disp,
2794 int size_in_memory)
2795 {
2796 Address addr = mem2address(opcode, base, index, scale, disp);
2797 if (addr.getMode() == Address::base_plus_offset) {
2798 /* Fix up any out-of-range offsets. */
2799 assert_different_registers(rscratch1, base);
2800 assert_different_registers(rscratch1, reg);
2801 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2802 }
2803 (masm->*insn)(reg, addr);
2804 }
2805
2806 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2807 FloatRegister reg, int opcode,
2808 Register base, int index, int size, int disp,
2809 int size_in_memory)
2810 {
2811 Address::extend scale;
2812
2813 switch (opcode) {
2814 case INDINDEXSCALEDI2L:
2815 case INDINDEXSCALEDI2LN:
2816 scale = Address::sxtw(size);
2817 break;
2818 default:
2819 scale = Address::lsl(size);
2820 }
2821
2822 if (index == -1) {
2823 // Fix up any out-of-range offsets.
2824 assert_different_registers(rscratch1, base);
2825 Address addr = Address(base, disp);
2826 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2827 (masm->*insn)(reg, addr);
2828 } else {
2829 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2830 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2831 }
2832 }
2833
2834 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2835 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2836 int opcode, Register base, int index, int size, int disp)
2837 {
2838 if (index == -1) {
2839 (masm->*insn)(reg, T, Address(base, disp));
2840 } else {
2841 assert(disp == 0, "unsupported address mode");
2842 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2843 }
2844 }
2845
2846 %}
2847
2848
2849
2850 //----------ENCODING BLOCK-----------------------------------------------------
2851 // This block specifies the encoding classes used by the compiler to
2852 // output byte streams. Encoding classes are parameterized macros
2853 // used by Machine Instruction Nodes in order to generate the bit
2854 // encoding of the instruction. Operands specify their base encoding
2855 // interface with the interface keyword. There are currently
2856 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2857 // COND_INTER. REG_INTER causes an operand to generate a function
2858 // which returns its register number when queried. CONST_INTER causes
2859 // an operand to generate a function which returns the value of the
2860 // constant when queried. MEMORY_INTER causes an operand to generate
2861 // four functions which return the Base Register, the Index Register,
2862 // the Scale Value, and the Offset Value of the operand when queried.
2863 // COND_INTER causes an operand to generate six functions which return
2864 // the encoding code (ie - encoding bits for the instruction)
2865 // associated with each basic boolean condition for a conditional
2866 // instruction.
2867 //
2868 // Instructions specify two basic values for encoding. Again, a
2869 // function is available to check if the constant displacement is an
2870 // oop. They use the ins_encode keyword to specify their encoding
2871 // classes (which must be a sequence of enc_class names, and their
2872 // parameters, specified in the encoding block), and they use the
2873 // opcode keyword to specify, in order, their primary, secondary, and
2874 // tertiary opcode. Only the opcode sections which a particular
2875 // instruction needs for encoding need to be specified.
2876 encode %{
2877 // Build emit functions for each basic byte or larger field in the
2878 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2879 // from C++ code in the enc_class source block. Emit functions will
2880 // live in the main source block for now. In future, we can
2881 // generalize this by adding a syntax that specifies the sizes of
2882 // fields in an order, so that the adlc can build the emit functions
2883 // automagically
2884
2885 // catch all for unimplemented encodings
2886 enc_class enc_unimplemented %{
2887 __ unimplemented("C2 catch all");
2888 %}
2889
2890 // BEGIN Non-volatile memory access
2891
2892 // This encoding class is generated automatically from ad_encode.m4.
2893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2894 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2895 Register dst_reg = as_Register($dst$$reg);
2896 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2897 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2898 %}
2899
2900 // This encoding class is generated automatically from ad_encode.m4.
2901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2902 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2903 Register dst_reg = as_Register($dst$$reg);
2904 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2905 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2906 %}
2907
2908 // This encoding class is generated automatically from ad_encode.m4.
2909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2910 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2911 Register dst_reg = as_Register($dst$$reg);
2912 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2913 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2914 %}
2915
2916 // This encoding class is generated automatically from ad_encode.m4.
2917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2918 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2919 Register dst_reg = as_Register($dst$$reg);
2920 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2921 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2922 %}
2923
2924 // This encoding class is generated automatically from ad_encode.m4.
2925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2926 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2927 Register dst_reg = as_Register($dst$$reg);
2928 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2929 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2930 %}
2931
2932 // This encoding class is generated automatically from ad_encode.m4.
2933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2934 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2935 Register dst_reg = as_Register($dst$$reg);
2936 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2937 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2938 %}
2939
2940 // This encoding class is generated automatically from ad_encode.m4.
2941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2942 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2943 Register dst_reg = as_Register($dst$$reg);
2944 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2945 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2946 %}
2947
2948 // This encoding class is generated automatically from ad_encode.m4.
2949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2950 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2951 Register dst_reg = as_Register($dst$$reg);
2952 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2953 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2954 %}
2955
2956 // This encoding class is generated automatically from ad_encode.m4.
2957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2958 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2959 Register dst_reg = as_Register($dst$$reg);
2960 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2961 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2962 %}
2963
2964 // This encoding class is generated automatically from ad_encode.m4.
2965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2966 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2967 Register dst_reg = as_Register($dst$$reg);
2968 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2969 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2970 %}
2971
2972 // This encoding class is generated automatically from ad_encode.m4.
2973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2974 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2975 Register dst_reg = as_Register($dst$$reg);
2976 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2977 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2978 %}
2979
2980 // This encoding class is generated automatically from ad_encode.m4.
2981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2982 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2983 Register dst_reg = as_Register($dst$$reg);
2984 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2985 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2986 %}
2987
2988 // This encoding class is generated automatically from ad_encode.m4.
2989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2990 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2991 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2992 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2993 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2994 %}
2995
2996 // This encoding class is generated automatically from ad_encode.m4.
2997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2998 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2999 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3000 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3001 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3002 %}
3003
3004 // This encoding class is generated automatically from ad_encode.m4.
3005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3006 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3007 Register src_reg = as_Register($src$$reg);
3008 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3009 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3010 %}
3011
3012 // This encoding class is generated automatically from ad_encode.m4.
3013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3014 enc_class aarch64_enc_strb0(memory1 mem) %{
3015 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3022 Register src_reg = as_Register($src$$reg);
3023 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strh0(memory2 mem) %{
3030 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3037 Register src_reg = as_Register($src$$reg);
3038 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_strw0(memory4 mem) %{
3045 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3046 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3047 %}
3048
3049 // This encoding class is generated automatically from ad_encode.m4.
3050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3051 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3052 Register src_reg = as_Register($src$$reg);
3053 // we sometimes get asked to store the stack pointer into the
3054 // current thread -- we cannot do that directly on AArch64
3055 if (src_reg == r31_sp) {
3056 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3057 __ mov(rscratch2, sp);
3058 src_reg = rscratch2;
3059 }
3060 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_str0(memory8 mem) %{
3067 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3068 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3069 %}
3070
3071 // This encoding class is generated automatically from ad_encode.m4.
3072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3073 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3074 FloatRegister src_reg = as_FloatRegister($src$$reg);
3075 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3076 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3077 %}
3078
3079 // This encoding class is generated automatically from ad_encode.m4.
3080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3081 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3082 FloatRegister src_reg = as_FloatRegister($src$$reg);
3083 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3084 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3085 %}
3086
3087 // This encoding class is generated automatically from ad_encode.m4.
3088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3089 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3090 __ membar(Assembler::StoreStore);
3091 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3092 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3093 %}
3094
3095 // END Non-volatile memory access
3096
3097 // Vector loads and stores
3098 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3099 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3100 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3101 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3102 %}
3103
3104 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3105 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3106 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3107 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3108 %}
3109
3110 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3111 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3112 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3113 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3114 %}
3115
3116 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3117 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3118 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3119 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3120 %}
3121
3122 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3123 FloatRegister src_reg = as_FloatRegister($src$$reg);
3124 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3125 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3126 %}
3127
3128 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3129 FloatRegister src_reg = as_FloatRegister($src$$reg);
3130 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3131 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3132 %}
3133
3134 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3135 FloatRegister src_reg = as_FloatRegister($src$$reg);
3136 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3137 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3138 %}
3139
3140 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3141 FloatRegister src_reg = as_FloatRegister($src$$reg);
3142 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3143 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3144 %}
3145
3146 // volatile loads and stores
3147
3148 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3149 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3150 rscratch1, stlrb);
3151 %}
3152
3153 enc_class aarch64_enc_stlrb0(memory mem) %{
3154 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, stlrb);
3156 %}
3157
3158 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3159 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, stlrh);
3161 %}
3162
3163 enc_class aarch64_enc_stlrh0(memory mem) %{
3164 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, stlrh);
3166 %}
3167
3168 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3169 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3170 rscratch1, stlrw);
3171 %}
3172
3173 enc_class aarch64_enc_stlrw0(memory mem) %{
3174 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, stlrw);
3176 %}
3177
3178 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3179 Register dst_reg = as_Register($dst$$reg);
3180 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarb);
3182 __ sxtbw(dst_reg, dst_reg);
3183 %}
3184
3185 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3186 Register dst_reg = as_Register($dst$$reg);
3187 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarb);
3189 __ sxtb(dst_reg, dst_reg);
3190 %}
3191
3192 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3193 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarb);
3195 %}
3196
3197 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3198 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, ldarb);
3200 %}
3201
3202 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3203 Register dst_reg = as_Register($dst$$reg);
3204 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarh);
3206 __ sxthw(dst_reg, dst_reg);
3207 %}
3208
3209 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3210 Register dst_reg = as_Register($dst$$reg);
3211 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, ldarh);
3213 __ sxth(dst_reg, dst_reg);
3214 %}
3215
3216 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3217 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, ldarh);
3219 %}
3220
3221 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3222 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3223 rscratch1, ldarh);
3224 %}
3225
3226 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3227 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3228 rscratch1, ldarw);
3229 %}
3230
3231 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3232 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3233 rscratch1, ldarw);
3234 %}
3235
3236 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3237 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, ldar);
3239 %}
3240
3241 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3242 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3243 rscratch1, ldarw);
3244 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3245 %}
3246
3247 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3248 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3249 rscratch1, ldar);
3250 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3251 %}
3252
3253 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3254 Register src_reg = as_Register($src$$reg);
3255 // we sometimes get asked to store the stack pointer into the
3256 // current thread -- we cannot do that directly on AArch64
3257 if (src_reg == r31_sp) {
3258 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3259 __ mov(rscratch2, sp);
3260 src_reg = rscratch2;
3261 }
3262 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, stlr);
3264 %}
3265
3266 enc_class aarch64_enc_stlr0(memory mem) %{
3267 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3268 rscratch1, stlr);
3269 %}
3270
3271 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3272 {
3273 FloatRegister src_reg = as_FloatRegister($src$$reg);
3274 __ fmovs(rscratch2, src_reg);
3275 }
3276 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, stlrw);
3278 %}
3279
3280 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3281 {
3282 FloatRegister src_reg = as_FloatRegister($src$$reg);
3283 __ fmovd(rscratch2, src_reg);
3284 }
3285 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3286 rscratch1, stlr);
3287 %}
3288
3289 // synchronized read/update encodings
3290
3291 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3292 Register dst_reg = as_Register($dst$$reg);
3293 Register base = as_Register($mem$$base);
3294 int index = $mem$$index;
3295 int scale = $mem$$scale;
3296 int disp = $mem$$disp;
3297 if (index == -1) {
3298 if (disp != 0) {
3299 __ lea(rscratch1, Address(base, disp));
3300 __ ldaxr(dst_reg, rscratch1);
3301 } else {
3302 // TODO
3303 // should we ever get anything other than this case?
3304 __ ldaxr(dst_reg, base);
3305 }
3306 } else {
3307 Register index_reg = as_Register(index);
3308 if (disp == 0) {
3309 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3310 __ ldaxr(dst_reg, rscratch1);
3311 } else {
3312 __ lea(rscratch1, Address(base, disp));
3313 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3314 __ ldaxr(dst_reg, rscratch1);
3315 }
3316 }
3317 %}
3318
3319 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3320 Register src_reg = as_Register($src$$reg);
3321 Register base = as_Register($mem$$base);
3322 int index = $mem$$index;
3323 int scale = $mem$$scale;
3324 int disp = $mem$$disp;
3325 if (index == -1) {
3326 if (disp != 0) {
3327 __ lea(rscratch2, Address(base, disp));
3328 __ stlxr(rscratch1, src_reg, rscratch2);
3329 } else {
3330 // TODO
3331 // should we ever get anything other than this case?
3332 __ stlxr(rscratch1, src_reg, base);
3333 }
3334 } else {
3335 Register index_reg = as_Register(index);
3336 if (disp == 0) {
3337 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3338 __ stlxr(rscratch1, src_reg, rscratch2);
3339 } else {
3340 __ lea(rscratch2, Address(base, disp));
3341 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3342 __ stlxr(rscratch1, src_reg, rscratch2);
3343 }
3344 }
3345 __ cmpw(rscratch1, zr);
3346 %}
3347
3348 // prefetch encodings
3349
3350 enc_class aarch64_enc_prefetchw(memory mem) %{
3351 Register base = as_Register($mem$$base);
3352 int index = $mem$$index;
3353 int scale = $mem$$scale;
3354 int disp = $mem$$disp;
3355 if (index == -1) {
3356 // Fix up any out-of-range offsets.
3357 assert_different_registers(rscratch1, base);
3358 Address addr = Address(base, disp);
3359 addr = __ legitimize_address(addr, 8, rscratch1);
3360 __ prfm(addr, PSTL1KEEP);
3361 } else {
3362 Register index_reg = as_Register(index);
3363 if (disp == 0) {
3364 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3365 } else {
3366 __ lea(rscratch1, Address(base, disp));
3367 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3368 }
3369 }
3370 %}
3371
3372 // mov encodings
3373
3374 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3375 uint32_t con = (uint32_t)$src$$constant;
3376 Register dst_reg = as_Register($dst$$reg);
3377 if (con == 0) {
3378 __ movw(dst_reg, zr);
3379 } else {
3380 __ movw(dst_reg, con);
3381 }
3382 %}
3383
3384 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3385 Register dst_reg = as_Register($dst$$reg);
3386 uint64_t con = (uint64_t)$src$$constant;
3387 if (con == 0) {
3388 __ mov(dst_reg, zr);
3389 } else {
3390 __ mov(dst_reg, con);
3391 }
3392 %}
3393
3394 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3395 Register dst_reg = as_Register($dst$$reg);
3396 address con = (address)$src$$constant;
3397 if (con == nullptr || con == (address)1) {
3398 ShouldNotReachHere();
3399 } else {
3400 relocInfo::relocType rtype = $src->constant_reloc();
3401 if (rtype == relocInfo::oop_type) {
3402 __ movoop(dst_reg, (jobject)con);
3403 } else if (rtype == relocInfo::metadata_type) {
3404 __ mov_metadata(dst_reg, (Metadata*)con);
3405 } else {
3406 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3407 if (! __ is_valid_AArch64_address(con) ||
3408 con < (address)(uintptr_t)os::vm_page_size()) {
3409 __ mov(dst_reg, con);
3410 } else {
3411 uint64_t offset;
3412 __ adrp(dst_reg, con, offset);
3413 __ add(dst_reg, dst_reg, offset);
3414 }
3415 }
3416 }
3417 %}
3418
3419 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3420 Register dst_reg = as_Register($dst$$reg);
3421 __ mov(dst_reg, zr);
3422 %}
3423
3424 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3425 Register dst_reg = as_Register($dst$$reg);
3426 __ mov(dst_reg, (uint64_t)1);
3427 %}
3428
3429 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3430 Register dst_reg = as_Register($dst$$reg);
3431 address con = (address)$src$$constant;
3432 if (con == nullptr) {
3433 ShouldNotReachHere();
3434 } else {
3435 relocInfo::relocType rtype = $src->constant_reloc();
3436 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3437 __ set_narrow_oop(dst_reg, (jobject)con);
3438 }
3439 %}
3440
3441 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3442 Register dst_reg = as_Register($dst$$reg);
3443 __ mov(dst_reg, zr);
3444 %}
3445
3446 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3447 Register dst_reg = as_Register($dst$$reg);
3448 address con = (address)$src$$constant;
3449 if (con == nullptr) {
3450 ShouldNotReachHere();
3451 } else {
3452 relocInfo::relocType rtype = $src->constant_reloc();
3453 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3454 __ set_narrow_klass(dst_reg, (Klass *)con);
3455 }
3456 %}
3457
3458 // arithmetic encodings
3459
3460 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3461 Register dst_reg = as_Register($dst$$reg);
3462 Register src_reg = as_Register($src1$$reg);
3463 int32_t con = (int32_t)$src2$$constant;
3464 // add has primary == 0, subtract has primary == 1
3465 if ($primary) { con = -con; }
3466 if (con < 0) {
3467 __ subw(dst_reg, src_reg, -con);
3468 } else {
3469 __ addw(dst_reg, src_reg, con);
3470 }
3471 %}
3472
3473 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3474 Register dst_reg = as_Register($dst$$reg);
3475 Register src_reg = as_Register($src1$$reg);
3476 int32_t con = (int32_t)$src2$$constant;
3477 // add has primary == 0, subtract has primary == 1
3478 if ($primary) { con = -con; }
3479 if (con < 0) {
3480 __ sub(dst_reg, src_reg, -con);
3481 } else {
3482 __ add(dst_reg, src_reg, con);
3483 }
3484 %}
3485
3486 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src1_reg = as_Register($src1$$reg);
3489 Register src2_reg = as_Register($src2$$reg);
3490 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3491 %}
3492
3493 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3494 Register dst_reg = as_Register($dst$$reg);
3495 Register src1_reg = as_Register($src1$$reg);
3496 Register src2_reg = as_Register($src2$$reg);
3497 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3498 %}
3499
3500 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src1_reg = as_Register($src1$$reg);
3503 Register src2_reg = as_Register($src2$$reg);
3504 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3505 %}
3506
3507 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3508 Register dst_reg = as_Register($dst$$reg);
3509 Register src1_reg = as_Register($src1$$reg);
3510 Register src2_reg = as_Register($src2$$reg);
3511 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3512 %}
3513
3514 // compare instruction encodings
3515
3516 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3517 Register reg1 = as_Register($src1$$reg);
3518 Register reg2 = as_Register($src2$$reg);
3519 __ cmpw(reg1, reg2);
3520 %}
3521
3522 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3523 Register reg = as_Register($src1$$reg);
3524 int32_t val = $src2$$constant;
3525 if (val >= 0) {
3526 __ subsw(zr, reg, val);
3527 } else {
3528 __ addsw(zr, reg, -val);
3529 }
3530 %}
3531
3532 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3533 Register reg1 = as_Register($src1$$reg);
3534 uint32_t val = (uint32_t)$src2$$constant;
3535 __ movw(rscratch1, val);
3536 __ cmpw(reg1, rscratch1);
3537 %}
3538
3539 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3540 Register reg1 = as_Register($src1$$reg);
3541 Register reg2 = as_Register($src2$$reg);
3542 __ cmp(reg1, reg2);
3543 %}
3544
3545 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3546 Register reg = as_Register($src1$$reg);
3547 int64_t val = $src2$$constant;
3548 if (val >= 0) {
3549 __ subs(zr, reg, val);
3550 } else if (val != -val) {
3551 __ adds(zr, reg, -val);
3552 } else {
3553 // aargh, Long.MIN_VALUE is a special case
3554 __ orr(rscratch1, zr, (uint64_t)val);
3555 __ subs(zr, reg, rscratch1);
3556 }
3557 %}
3558
3559 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3560 Register reg1 = as_Register($src1$$reg);
3561 uint64_t val = (uint64_t)$src2$$constant;
3562 __ mov(rscratch1, val);
3563 __ cmp(reg1, rscratch1);
3564 %}
3565
3566 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3567 Register reg1 = as_Register($src1$$reg);
3568 Register reg2 = as_Register($src2$$reg);
3569 __ cmp(reg1, reg2);
3570 %}
3571
3572 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3573 Register reg1 = as_Register($src1$$reg);
3574 Register reg2 = as_Register($src2$$reg);
3575 __ cmpw(reg1, reg2);
3576 %}
3577
3578 enc_class aarch64_enc_testp(iRegP src) %{
3579 Register reg = as_Register($src$$reg);
3580 __ cmp(reg, zr);
3581 %}
3582
3583 enc_class aarch64_enc_testn(iRegN src) %{
3584 Register reg = as_Register($src$$reg);
3585 __ cmpw(reg, zr);
3586 %}
3587
3588 enc_class aarch64_enc_b(label lbl) %{
3589 Label *L = $lbl$$label;
3590 __ b(*L);
3591 %}
3592
3593 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3594 Label *L = $lbl$$label;
3595 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3596 %}
3597
3598 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3599 Label *L = $lbl$$label;
3600 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3601 %}
3602
3603 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3604 %{
3605 Register sub_reg = as_Register($sub$$reg);
3606 Register super_reg = as_Register($super$$reg);
3607 Register temp_reg = as_Register($temp$$reg);
3608 Register result_reg = as_Register($result$$reg);
3609
3610 Label miss;
3611 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3612 nullptr, &miss,
3613 /*set_cond_codes:*/ true);
3614 if ($primary) {
3615 __ mov(result_reg, zr);
3616 }
3617 __ bind(miss);
3618 %}
3619
3620 enc_class aarch64_enc_java_static_call(method meth) %{
3621 address addr = (address)$meth$$method;
3622 address call;
3623 if (!_method) {
3624 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3625 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3626 if (call == nullptr) {
3627 ciEnv::current()->record_failure("CodeCache is full");
3628 return;
3629 }
3630 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3631 // The NOP here is purely to ensure that eliding a call to
3632 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3633 __ nop();
3634 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3635 } else {
3636 int method_index = resolved_method_index(masm);
3637 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3638 : static_call_Relocation::spec(method_index);
3639 call = __ trampoline_call(Address(addr, rspec));
3640 if (call == nullptr) {
3641 ciEnv::current()->record_failure("CodeCache is full");
3642 return;
3643 }
3644 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3645 // Calls of the same statically bound method can share
3646 // a stub to the interpreter.
3647 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3648 } else {
3649 // Emit stub for static call
3650 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3651 if (stub == nullptr) {
3652 ciEnv::current()->record_failure("CodeCache is full");
3653 return;
3654 }
3655 }
3656 }
3657
3658 __ post_call_nop();
3659
3660 // Only non uncommon_trap calls need to reinitialize ptrue.
3661 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3662 __ reinitialize_ptrue();
3663 }
3664 %}
3665
3666 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3667 int method_index = resolved_method_index(masm);
3668 address call = __ ic_call((address)$meth$$method, method_index);
3669 if (call == nullptr) {
3670 ciEnv::current()->record_failure("CodeCache is full");
3671 return;
3672 }
3673 __ post_call_nop();
3674 if (Compile::current()->max_vector_size() > 0) {
3675 __ reinitialize_ptrue();
3676 }
3677 %}
3678
3679 enc_class aarch64_enc_call_epilog() %{
3680 if (VerifyStackAtCalls) {
3681 // Check that stack depth is unchanged: find majik cookie on stack
3682 __ call_Unimplemented();
3683 }
3684 %}
3685
3686 enc_class aarch64_enc_java_to_runtime(method meth) %{
3687 // some calls to generated routines (arraycopy code) are scheduled
3688 // by C2 as runtime calls. if so we can call them using a br (they
3689 // will be in a reachable segment) otherwise we have to use a blr
3690 // which loads the absolute address into a register.
3691 address entry = (address)$meth$$method;
3692 CodeBlob *cb = CodeCache::find_blob(entry);
3693 if (cb) {
3694 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3695 if (call == nullptr) {
3696 ciEnv::current()->record_failure("CodeCache is full");
3697 return;
3698 }
3699 __ post_call_nop();
3700 } else {
3701 Label retaddr;
3702 // Make the anchor frame walkable
3703 __ adr(rscratch2, retaddr);
3704 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3705 __ lea(rscratch1, RuntimeAddress(entry));
3706 __ blr(rscratch1);
3707 __ bind(retaddr);
3708 __ post_call_nop();
3709 }
3710 if (Compile::current()->max_vector_size() > 0) {
3711 __ reinitialize_ptrue();
3712 }
3713 %}
3714
3715 enc_class aarch64_enc_rethrow() %{
3716 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3717 %}
3718
3719 enc_class aarch64_enc_ret() %{
3720 #ifdef ASSERT
3721 if (Compile::current()->max_vector_size() > 0) {
3722 __ verify_ptrue();
3723 }
3724 #endif
3725 __ ret(lr);
3726 %}
3727
3728 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3729 Register target_reg = as_Register($jump_target$$reg);
3730 __ br(target_reg);
3731 %}
3732
3733 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3734 Register target_reg = as_Register($jump_target$$reg);
3735 // exception oop should be in r0
3736 // ret addr has been popped into lr
3737 // callee expects it in r3
3738 __ mov(r3, lr);
3739 __ br(target_reg);
3740 %}
3741
3742 %}
3743
3744 //----------FRAME--------------------------------------------------------------
3745 // Definition of frame structure and management information.
3746 //
3747 // S T A C K L A Y O U T Allocators stack-slot number
3748 // | (to get allocators register number
3749 // G Owned by | | v add OptoReg::stack0())
3750 // r CALLER | |
3751 // o | +--------+ pad to even-align allocators stack-slot
3752 // w V | pad0 | numbers; owned by CALLER
3753 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3754 // h ^ | in | 5
3755 // | | args | 4 Holes in incoming args owned by SELF
3756 // | | | | 3
3757 // | | +--------+
3758 // V | | old out| Empty on Intel, window on Sparc
3759 // | old |preserve| Must be even aligned.
3760 // | SP-+--------+----> Matcher::_old_SP, even aligned
3761 // | | in | 3 area for Intel ret address
3762 // Owned by |preserve| Empty on Sparc.
3763 // SELF +--------+
3764 // | | pad2 | 2 pad to align old SP
3765 // | +--------+ 1
3766 // | | locks | 0
3767 // | +--------+----> OptoReg::stack0(), even aligned
3768 // | | pad1 | 11 pad to align new SP
3769 // | +--------+
3770 // | | | 10
3771 // | | spills | 9 spills
3772 // V | | 8 (pad0 slot for callee)
3773 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3774 // ^ | out | 7
3775 // | | args | 6 Holes in outgoing args owned by CALLEE
3776 // Owned by +--------+
3777 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3778 // | new |preserve| Must be even-aligned.
3779 // | SP-+--------+----> Matcher::_new_SP, even aligned
3780 // | | |
3781 //
3782 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3783 // known from SELF's arguments and the Java calling convention.
3784 // Region 6-7 is determined per call site.
3785 // Note 2: If the calling convention leaves holes in the incoming argument
3786 // area, those holes are owned by SELF. Holes in the outgoing area
3787 // are owned by the CALLEE. Holes should not be necessary in the
3788 // incoming area, as the Java calling convention is completely under
3789 // the control of the AD file. Doubles can be sorted and packed to
3790 // avoid holes. Holes in the outgoing arguments may be necessary for
3791 // varargs C calling conventions.
3792 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3793 // even aligned with pad0 as needed.
3794 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3795 // (the latter is true on Intel but is it false on AArch64?)
3796 // region 6-11 is even aligned; it may be padded out more so that
3797 // the region from SP to FP meets the minimum stack alignment.
3798 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3799 // alignment. Region 11, pad1, may be dynamically extended so that
3800 // SP meets the minimum alignment.
3801
3802 frame %{
3803 // These three registers define part of the calling convention
3804 // between compiled code and the interpreter.
3805
3806 // Inline Cache Register or Method for I2C.
3807 inline_cache_reg(R12);
3808
3809 // Number of stack slots consumed by locking an object
3810 sync_stack_slots(2);
3811
3812 // Compiled code's Frame Pointer
3813 frame_pointer(R31);
3814
3815 // Interpreter stores its frame pointer in a register which is
3816 // stored to the stack by I2CAdaptors.
3817 // I2CAdaptors convert from interpreted java to compiled java.
3818 interpreter_frame_pointer(R29);
3819
3820 // Stack alignment requirement
3821 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3822
3823 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3824 // for calls to C. Supports the var-args backing area for register parms.
3825 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3826
3827 // The after-PROLOG location of the return address. Location of
3828 // return address specifies a type (REG or STACK) and a number
3829 // representing the register number (i.e. - use a register name) or
3830 // stack slot.
3831 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3832 // Otherwise, it is above the locks and verification slot and alignment word
3833 // TODO this may well be correct but need to check why that - 2 is there
3834 // ppc port uses 0 but we definitely need to allow for fixed_slots
3835 // which folds in the space used for monitors
3836 return_addr(STACK - 2 +
3837 align_up((Compile::current()->in_preserve_stack_slots() +
3838 Compile::current()->fixed_slots()),
3839 stack_alignment_in_slots()));
3840
3841 // Location of compiled Java return values. Same as C for now.
3842 return_value
3843 %{
3844 // TODO do we allow ideal_reg == Op_RegN???
3845 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3846 "only return normal values");
3847
3848 static const int lo[Op_RegL + 1] = { // enum name
3849 0, // Op_Node
3850 0, // Op_Set
3851 R0_num, // Op_RegN
3852 R0_num, // Op_RegI
3853 R0_num, // Op_RegP
3854 V0_num, // Op_RegF
3855 V0_num, // Op_RegD
3856 R0_num // Op_RegL
3857 };
3858
3859 static const int hi[Op_RegL + 1] = { // enum name
3860 0, // Op_Node
3861 0, // Op_Set
3862 OptoReg::Bad, // Op_RegN
3863 OptoReg::Bad, // Op_RegI
3864 R0_H_num, // Op_RegP
3865 OptoReg::Bad, // Op_RegF
3866 V0_H_num, // Op_RegD
3867 R0_H_num // Op_RegL
3868 };
3869
3870 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3871 %}
3872 %}
3873
3874 //----------ATTRIBUTES---------------------------------------------------------
3875 //----------Operand Attributes-------------------------------------------------
3876 op_attrib op_cost(1); // Required cost attribute
3877
3878 //----------Instruction Attributes---------------------------------------------
3879 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3880 ins_attrib ins_size(32); // Required size attribute (in bits)
3881 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3882 // a non-matching short branch variant
3883 // of some long branch?
3884 ins_attrib ins_alignment(4); // Required alignment attribute (must
3885 // be a power of 2) specifies the
3886 // alignment that some part of the
3887 // instruction (not necessarily the
3888 // start) requires. If > 1, a
3889 // compute_padding() function must be
3890 // provided for the instruction
3891
3892 // Whether this node is expanded during code emission into a sequence of
3893 // instructions and the first instruction can perform an implicit null check.
3894 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3895
3896 //----------OPERANDS-----------------------------------------------------------
3897 // Operand definitions must precede instruction definitions for correct parsing
3898 // in the ADLC because operands constitute user defined types which are used in
3899 // instruction definitions.
3900
3901 //----------Simple Operands----------------------------------------------------
3902
3903 // Integer operands 32 bit
3904 // 32 bit immediate
3905 operand immI()
3906 %{
3907 match(ConI);
3908
3909 op_cost(0);
3910 format %{ %}
3911 interface(CONST_INTER);
3912 %}
3913
3914 // 32 bit zero
3915 operand immI0()
3916 %{
3917 predicate(n->get_int() == 0);
3918 match(ConI);
3919
3920 op_cost(0);
3921 format %{ %}
3922 interface(CONST_INTER);
3923 %}
3924
3925 // 32 bit unit increment
3926 operand immI_1()
3927 %{
3928 predicate(n->get_int() == 1);
3929 match(ConI);
3930
3931 op_cost(0);
3932 format %{ %}
3933 interface(CONST_INTER);
3934 %}
3935
3936 // 32 bit unit decrement
3937 operand immI_M1()
3938 %{
3939 predicate(n->get_int() == -1);
3940 match(ConI);
3941
3942 op_cost(0);
3943 format %{ %}
3944 interface(CONST_INTER);
3945 %}
3946
3947 // Shift values for add/sub extension shift
3948 operand immIExt()
3949 %{
3950 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_gt_1()
3959 %{
3960 predicate(n->get_int() > 1);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_le_4()
3969 %{
3970 predicate(n->get_int() <= 4);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_16()
3979 %{
3980 predicate(n->get_int() == 16);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_24()
3989 %{
3990 predicate(n->get_int() == 24);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_32()
3999 %{
4000 predicate(n->get_int() == 32);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_48()
4009 %{
4010 predicate(n->get_int() == 48);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_56()
4019 %{
4020 predicate(n->get_int() == 56);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_255()
4029 %{
4030 predicate(n->get_int() == 255);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_65535()
4039 %{
4040 predicate(n->get_int() == 65535);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 operand immI_positive()
4049 %{
4050 predicate(n->get_int() > 0);
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 // BoolTest condition for signed compare
4059 operand immI_cmp_cond()
4060 %{
4061 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4062 match(ConI);
4063
4064 op_cost(0);
4065 format %{ %}
4066 interface(CONST_INTER);
4067 %}
4068
4069 // BoolTest condition for unsigned compare
4070 operand immI_cmpU_cond()
4071 %{
4072 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4073 match(ConI);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immL_255()
4081 %{
4082 predicate(n->get_long() == 255L);
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_65535()
4091 %{
4092 predicate(n->get_long() == 65535L);
4093 match(ConL);
4094
4095 op_cost(0);
4096 format %{ %}
4097 interface(CONST_INTER);
4098 %}
4099
4100 operand immL_4294967295()
4101 %{
4102 predicate(n->get_long() == 4294967295L);
4103 match(ConL);
4104
4105 op_cost(0);
4106 format %{ %}
4107 interface(CONST_INTER);
4108 %}
4109
4110 operand immL_bitmask()
4111 %{
4112 predicate((n->get_long() != 0)
4113 && ((n->get_long() & 0xc000000000000000l) == 0)
4114 && is_power_of_2(n->get_long() + 1));
4115 match(ConL);
4116
4117 op_cost(0);
4118 format %{ %}
4119 interface(CONST_INTER);
4120 %}
4121
4122 operand immI_bitmask()
4123 %{
4124 predicate((n->get_int() != 0)
4125 && ((n->get_int() & 0xc0000000) == 0)
4126 && is_power_of_2(n->get_int() + 1));
4127 match(ConI);
4128
4129 op_cost(0);
4130 format %{ %}
4131 interface(CONST_INTER);
4132 %}
4133
4134 operand immL_positive_bitmaskI()
4135 %{
4136 predicate((n->get_long() != 0)
4137 && ((julong)n->get_long() < 0x80000000ULL)
4138 && is_power_of_2(n->get_long() + 1));
4139 match(ConL);
4140
4141 op_cost(0);
4142 format %{ %}
4143 interface(CONST_INTER);
4144 %}
4145
4146 // Scale values for scaled offset addressing modes (up to long but not quad)
4147 operand immIScale()
4148 %{
4149 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4150 match(ConI);
4151
4152 op_cost(0);
4153 format %{ %}
4154 interface(CONST_INTER);
4155 %}
4156
4157 // 5 bit signed integer
4158 operand immI5()
4159 %{
4160 predicate(Assembler::is_simm(n->get_int(), 5));
4161 match(ConI);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 // 7 bit unsigned integer
4169 operand immIU7()
4170 %{
4171 predicate(Assembler::is_uimm(n->get_int(), 7));
4172 match(ConI);
4173
4174 op_cost(0);
4175 format %{ %}
4176 interface(CONST_INTER);
4177 %}
4178
4179 // Offset for scaled or unscaled immediate loads and stores
4180 operand immIOffset()
4181 %{
4182 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immIOffset1()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset2()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset4()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immIOffset8()
4221 %{
4222 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immIOffset16()
4231 %{
4232 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immLOffset()
4241 %{
4242 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4243 match(ConL);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLoffset1()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset2()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset4()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immLoffset8()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immLoffset16()
4291 %{
4292 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4293 match(ConL);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 // 5 bit signed long integer
4301 operand immL5()
4302 %{
4303 predicate(Assembler::is_simm(n->get_long(), 5));
4304 match(ConL);
4305
4306 op_cost(0);
4307 format %{ %}
4308 interface(CONST_INTER);
4309 %}
4310
4311 // 7 bit unsigned long integer
4312 operand immLU7()
4313 %{
4314 predicate(Assembler::is_uimm(n->get_long(), 7));
4315 match(ConL);
4316
4317 op_cost(0);
4318 format %{ %}
4319 interface(CONST_INTER);
4320 %}
4321
4322 // 8 bit signed value.
4323 operand immI8()
4324 %{
4325 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4326 match(ConI);
4327
4328 op_cost(0);
4329 format %{ %}
4330 interface(CONST_INTER);
4331 %}
4332
4333 // 8 bit signed value (simm8), or #simm8 LSL 8.
4334 operand immIDupV()
4335 %{
4336 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4337 match(ConI);
4338
4339 op_cost(0);
4340 format %{ %}
4341 interface(CONST_INTER);
4342 %}
4343
4344 // 8 bit signed value (simm8), or #simm8 LSL 8.
4345 operand immLDupV()
4346 %{
4347 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4348 match(ConL);
4349
4350 op_cost(0);
4351 format %{ %}
4352 interface(CONST_INTER);
4353 %}
4354
4355 // 8 bit signed value (simm8), or #simm8 LSL 8.
4356 operand immHDupV()
4357 %{
4358 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4359 match(ConH);
4360
4361 op_cost(0);
4362 format %{ %}
4363 interface(CONST_INTER);
4364 %}
4365
4366 // 8 bit integer valid for vector add sub immediate
4367 operand immBAddSubV()
4368 %{
4369 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4370 match(ConI);
4371
4372 op_cost(0);
4373 format %{ %}
4374 interface(CONST_INTER);
4375 %}
4376
4377 // 32 bit integer valid for add sub immediate
4378 operand immIAddSub()
4379 %{
4380 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4381 match(ConI);
4382 op_cost(0);
4383 format %{ %}
4384 interface(CONST_INTER);
4385 %}
4386
4387 // 32 bit integer valid for vector add sub immediate
4388 operand immIAddSubV()
4389 %{
4390 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4391 match(ConI);
4392
4393 op_cost(0);
4394 format %{ %}
4395 interface(CONST_INTER);
4396 %}
4397
4398 // 32 bit unsigned integer valid for logical immediate
4399
4400 operand immBLog()
4401 %{
4402 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 operand immSLog()
4411 %{
4412 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4413 match(ConI);
4414
4415 op_cost(0);
4416 format %{ %}
4417 interface(CONST_INTER);
4418 %}
4419
4420 operand immILog()
4421 %{
4422 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4423 match(ConI);
4424
4425 op_cost(0);
4426 format %{ %}
4427 interface(CONST_INTER);
4428 %}
4429
4430 // Integer operands 64 bit
4431 // 64 bit immediate
4432 operand immL()
4433 %{
4434 match(ConL);
4435
4436 op_cost(0);
4437 format %{ %}
4438 interface(CONST_INTER);
4439 %}
4440
4441 // 64 bit zero
4442 operand immL0()
4443 %{
4444 predicate(n->get_long() == 0);
4445 match(ConL);
4446
4447 op_cost(0);
4448 format %{ %}
4449 interface(CONST_INTER);
4450 %}
4451
4452 // 64 bit unit decrement
4453 operand immL_M1()
4454 %{
4455 predicate(n->get_long() == -1);
4456 match(ConL);
4457
4458 op_cost(0);
4459 format %{ %}
4460 interface(CONST_INTER);
4461 %}
4462
4463 // 64 bit integer valid for add sub immediate
4464 operand immLAddSub()
4465 %{
4466 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4467 match(ConL);
4468 op_cost(0);
4469 format %{ %}
4470 interface(CONST_INTER);
4471 %}
4472
4473 // 64 bit integer valid for addv subv immediate
4474 operand immLAddSubV()
4475 %{
4476 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4477 match(ConL);
4478
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // 64 bit integer valid for logical immediate
4485 operand immLLog()
4486 %{
4487 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4488 match(ConL);
4489 op_cost(0);
4490 format %{ %}
4491 interface(CONST_INTER);
4492 %}
4493
4494 // Long Immediate: low 32-bit mask
4495 operand immL_32bits()
4496 %{
4497 predicate(n->get_long() == 0xFFFFFFFFL);
4498 match(ConL);
4499 op_cost(0);
4500 format %{ %}
4501 interface(CONST_INTER);
4502 %}
4503
4504 // Pointer operands
4505 // Pointer Immediate
4506 operand immP()
4507 %{
4508 match(ConP);
4509
4510 op_cost(0);
4511 format %{ %}
4512 interface(CONST_INTER);
4513 %}
4514
4515 // nullptr Pointer Immediate
4516 operand immP0()
4517 %{
4518 predicate(n->get_ptr() == 0);
4519 match(ConP);
4520
4521 op_cost(0);
4522 format %{ %}
4523 interface(CONST_INTER);
4524 %}
4525
4526 // Pointer Immediate One
4527 // this is used in object initialization (initial object header)
4528 operand immP_1()
4529 %{
4530 predicate(n->get_ptr() == 1);
4531 match(ConP);
4532
4533 op_cost(0);
4534 format %{ %}
4535 interface(CONST_INTER);
4536 %}
4537
4538 // AOT Runtime Constants Address
4539 operand immAOTRuntimeConstantsAddress()
4540 %{
4541 // Check if the address is in the range of AOT Runtime Constants
4542 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4543 match(ConP);
4544
4545 op_cost(0);
4546 format %{ %}
4547 interface(CONST_INTER);
4548 %}
4549
4550 // Float and Double operands
4551 // Double Immediate
4552 operand immD()
4553 %{
4554 match(ConD);
4555 op_cost(0);
4556 format %{ %}
4557 interface(CONST_INTER);
4558 %}
4559
4560 // Double Immediate: +0.0d
4561 operand immD0()
4562 %{
4563 predicate(jlong_cast(n->getd()) == 0);
4564 match(ConD);
4565
4566 op_cost(0);
4567 format %{ %}
4568 interface(CONST_INTER);
4569 %}
4570
4571 // constant 'double +0.0'.
4572 operand immDPacked()
4573 %{
4574 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4575 match(ConD);
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // Float Immediate
4582 operand immF()
4583 %{
4584 match(ConF);
4585 op_cost(0);
4586 format %{ %}
4587 interface(CONST_INTER);
4588 %}
4589
4590 // Float Immediate: +0.0f.
4591 operand immF0()
4592 %{
4593 predicate(jint_cast(n->getf()) == 0);
4594 match(ConF);
4595
4596 op_cost(0);
4597 format %{ %}
4598 interface(CONST_INTER);
4599 %}
4600
4601 // Half Float (FP16) Immediate
4602 operand immH()
4603 %{
4604 match(ConH);
4605 op_cost(0);
4606 format %{ %}
4607 interface(CONST_INTER);
4608 %}
4609
4610 //
4611 operand immFPacked()
4612 %{
4613 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4614 match(ConF);
4615 op_cost(0);
4616 format %{ %}
4617 interface(CONST_INTER);
4618 %}
4619
4620 // Narrow pointer operands
4621 // Narrow Pointer Immediate
4622 operand immN()
4623 %{
4624 match(ConN);
4625
4626 op_cost(0);
4627 format %{ %}
4628 interface(CONST_INTER);
4629 %}
4630
4631 // Narrow nullptr Pointer Immediate
4632 operand immN0()
4633 %{
4634 predicate(n->get_narrowcon() == 0);
4635 match(ConN);
4636
4637 op_cost(0);
4638 format %{ %}
4639 interface(CONST_INTER);
4640 %}
4641
4642 operand immNKlass()
4643 %{
4644 match(ConNKlass);
4645
4646 op_cost(0);
4647 format %{ %}
4648 interface(CONST_INTER);
4649 %}
4650
4651 // Integer 32 bit Register Operands
4652 // Integer 32 bitRegister (excludes SP)
4653 operand iRegI()
4654 %{
4655 constraint(ALLOC_IN_RC(any_reg32));
4656 match(RegI);
4657 match(iRegINoSp);
4658 op_cost(0);
4659 format %{ %}
4660 interface(REG_INTER);
4661 %}
4662
4663 // Integer 32 bit Register not Special
4664 operand iRegINoSp()
4665 %{
4666 constraint(ALLOC_IN_RC(no_special_reg32));
4667 match(RegI);
4668 op_cost(0);
4669 format %{ %}
4670 interface(REG_INTER);
4671 %}
4672
4673 // Integer 64 bit Register Operands
4674 // Integer 64 bit Register (includes SP)
4675 operand iRegL()
4676 %{
4677 constraint(ALLOC_IN_RC(any_reg));
4678 match(RegL);
4679 match(iRegLNoSp);
4680 op_cost(0);
4681 format %{ %}
4682 interface(REG_INTER);
4683 %}
4684
4685 // Integer 64 bit Register not Special
4686 operand iRegLNoSp()
4687 %{
4688 constraint(ALLOC_IN_RC(no_special_reg));
4689 match(RegL);
4690 match(iRegL_R0);
4691 format %{ %}
4692 interface(REG_INTER);
4693 %}
4694
4695 // Pointer Register Operands
4696 // Pointer Register
4697 operand iRegP()
4698 %{
4699 constraint(ALLOC_IN_RC(ptr_reg));
4700 match(RegP);
4701 match(iRegPNoSp);
4702 match(iRegP_R0);
4703 //match(iRegP_R2);
4704 //match(iRegP_R4);
4705 match(iRegP_R5);
4706 match(thread_RegP);
4707 op_cost(0);
4708 format %{ %}
4709 interface(REG_INTER);
4710 %}
4711
4712 // Pointer 64 bit Register not Special
4713 operand iRegPNoSp()
4714 %{
4715 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4716 match(RegP);
4717 // match(iRegP);
4718 // match(iRegP_R0);
4719 // match(iRegP_R2);
4720 // match(iRegP_R4);
4721 // match(iRegP_R5);
4722 // match(thread_RegP);
4723 op_cost(0);
4724 format %{ %}
4725 interface(REG_INTER);
4726 %}
4727
4728 // This operand is not allowed to use rfp even if
4729 // rfp is not used to hold the frame pointer.
4730 operand iRegPNoSpNoRfp()
4731 %{
4732 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4733 match(RegP);
4734 match(iRegPNoSp);
4735 op_cost(0);
4736 format %{ %}
4737 interface(REG_INTER);
4738 %}
4739
4740 // Pointer 64 bit Register R0 only
4741 operand iRegP_R0()
4742 %{
4743 constraint(ALLOC_IN_RC(r0_reg));
4744 match(RegP);
4745 // match(iRegP);
4746 match(iRegPNoSp);
4747 op_cost(0);
4748 format %{ %}
4749 interface(REG_INTER);
4750 %}
4751
4752 // Pointer 64 bit Register R1 only
4753 operand iRegP_R1()
4754 %{
4755 constraint(ALLOC_IN_RC(r1_reg));
4756 match(RegP);
4757 // match(iRegP);
4758 match(iRegPNoSp);
4759 op_cost(0);
4760 format %{ %}
4761 interface(REG_INTER);
4762 %}
4763
4764 // Pointer 64 bit Register R2 only
4765 operand iRegP_R2()
4766 %{
4767 constraint(ALLOC_IN_RC(r2_reg));
4768 match(RegP);
4769 // match(iRegP);
4770 match(iRegPNoSp);
4771 op_cost(0);
4772 format %{ %}
4773 interface(REG_INTER);
4774 %}
4775
4776 // Pointer 64 bit Register R3 only
4777 operand iRegP_R3()
4778 %{
4779 constraint(ALLOC_IN_RC(r3_reg));
4780 match(RegP);
4781 // match(iRegP);
4782 match(iRegPNoSp);
4783 op_cost(0);
4784 format %{ %}
4785 interface(REG_INTER);
4786 %}
4787
4788 // Pointer 64 bit Register R4 only
4789 operand iRegP_R4()
4790 %{
4791 constraint(ALLOC_IN_RC(r4_reg));
4792 match(RegP);
4793 // match(iRegP);
4794 match(iRegPNoSp);
4795 op_cost(0);
4796 format %{ %}
4797 interface(REG_INTER);
4798 %}
4799
4800 // Pointer 64 bit Register R5 only
4801 operand iRegP_R5()
4802 %{
4803 constraint(ALLOC_IN_RC(r5_reg));
4804 match(RegP);
4805 // match(iRegP);
4806 match(iRegPNoSp);
4807 op_cost(0);
4808 format %{ %}
4809 interface(REG_INTER);
4810 %}
4811
4812 // Pointer 64 bit Register R10 only
4813 operand iRegP_R10()
4814 %{
4815 constraint(ALLOC_IN_RC(r10_reg));
4816 match(RegP);
4817 // match(iRegP);
4818 match(iRegPNoSp);
4819 op_cost(0);
4820 format %{ %}
4821 interface(REG_INTER);
4822 %}
4823
4824 // Long 64 bit Register R0 only
4825 operand iRegL_R0()
4826 %{
4827 constraint(ALLOC_IN_RC(r0_reg));
4828 match(RegL);
4829 match(iRegLNoSp);
4830 op_cost(0);
4831 format %{ %}
4832 interface(REG_INTER);
4833 %}
4834
4835 // Long 64 bit Register R11 only
4836 operand iRegL_R11()
4837 %{
4838 constraint(ALLOC_IN_RC(r11_reg));
4839 match(RegL);
4840 match(iRegLNoSp);
4841 op_cost(0);
4842 format %{ %}
4843 interface(REG_INTER);
4844 %}
4845
4846 // Register R0 only
4847 operand iRegI_R0()
4848 %{
4849 constraint(ALLOC_IN_RC(int_r0_reg));
4850 match(RegI);
4851 match(iRegINoSp);
4852 op_cost(0);
4853 format %{ %}
4854 interface(REG_INTER);
4855 %}
4856
4857 // Register R2 only
4858 operand iRegI_R2()
4859 %{
4860 constraint(ALLOC_IN_RC(int_r2_reg));
4861 match(RegI);
4862 match(iRegINoSp);
4863 op_cost(0);
4864 format %{ %}
4865 interface(REG_INTER);
4866 %}
4867
4868 // Register R3 only
4869 operand iRegI_R3()
4870 %{
4871 constraint(ALLOC_IN_RC(int_r3_reg));
4872 match(RegI);
4873 match(iRegINoSp);
4874 op_cost(0);
4875 format %{ %}
4876 interface(REG_INTER);
4877 %}
4878
4879
4880 // Register R4 only
4881 operand iRegI_R4()
4882 %{
4883 constraint(ALLOC_IN_RC(int_r4_reg));
4884 match(RegI);
4885 match(iRegINoSp);
4886 op_cost(0);
4887 format %{ %}
4888 interface(REG_INTER);
4889 %}
4890
4891
4892 // Pointer Register Operands
4893 // Narrow Pointer Register
4894 operand iRegN()
4895 %{
4896 constraint(ALLOC_IN_RC(any_reg32));
4897 match(RegN);
4898 match(iRegNNoSp);
4899 op_cost(0);
4900 format %{ %}
4901 interface(REG_INTER);
4902 %}
4903
4904 // Integer 64 bit Register not Special
4905 operand iRegNNoSp()
4906 %{
4907 constraint(ALLOC_IN_RC(no_special_reg32));
4908 match(RegN);
4909 op_cost(0);
4910 format %{ %}
4911 interface(REG_INTER);
4912 %}
4913
4914 // Float Register
4915 // Float register operands
4916 operand vRegF()
4917 %{
4918 constraint(ALLOC_IN_RC(float_reg));
4919 match(RegF);
4920
4921 op_cost(0);
4922 format %{ %}
4923 interface(REG_INTER);
4924 %}
4925
4926 // Double Register
4927 // Double register operands
4928 operand vRegD()
4929 %{
4930 constraint(ALLOC_IN_RC(double_reg));
4931 match(RegD);
4932
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938 // Generic vector class. This will be used for
4939 // all vector operands, including NEON and SVE.
4940 operand vReg()
4941 %{
4942 constraint(ALLOC_IN_RC(dynamic));
4943 match(VecA);
4944 match(VecD);
4945 match(VecX);
4946
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 operand vReg_V10()
4953 %{
4954 constraint(ALLOC_IN_RC(v10_veca_reg));
4955 match(vReg);
4956
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vReg_V11()
4963 %{
4964 constraint(ALLOC_IN_RC(v11_veca_reg));
4965 match(vReg);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vReg_V12()
4973 %{
4974 constraint(ALLOC_IN_RC(v12_veca_reg));
4975 match(vReg);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vReg_V13()
4983 %{
4984 constraint(ALLOC_IN_RC(v13_veca_reg));
4985 match(vReg);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V17()
4993 %{
4994 constraint(ALLOC_IN_RC(v17_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V18()
5003 %{
5004 constraint(ALLOC_IN_RC(v18_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vReg_V23()
5013 %{
5014 constraint(ALLOC_IN_RC(v23_veca_reg));
5015 match(vReg);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vReg_V24()
5023 %{
5024 constraint(ALLOC_IN_RC(v24_veca_reg));
5025 match(vReg);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vecA()
5033 %{
5034 constraint(ALLOC_IN_RC(vectora_reg));
5035 match(VecA);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vecD()
5043 %{
5044 constraint(ALLOC_IN_RC(vectord_reg));
5045 match(VecD);
5046
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vecX()
5053 %{
5054 constraint(ALLOC_IN_RC(vectorx_reg));
5055 match(VecX);
5056
5057 op_cost(0);
5058 format %{ %}
5059 interface(REG_INTER);
5060 %}
5061
5062 operand vRegD_V0()
5063 %{
5064 constraint(ALLOC_IN_RC(v0_reg));
5065 match(RegD);
5066 op_cost(0);
5067 format %{ %}
5068 interface(REG_INTER);
5069 %}
5070
5071 operand vRegD_V1()
5072 %{
5073 constraint(ALLOC_IN_RC(v1_reg));
5074 match(RegD);
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 operand vRegD_V2()
5081 %{
5082 constraint(ALLOC_IN_RC(v2_reg));
5083 match(RegD);
5084 op_cost(0);
5085 format %{ %}
5086 interface(REG_INTER);
5087 %}
5088
5089 operand vRegD_V3()
5090 %{
5091 constraint(ALLOC_IN_RC(v3_reg));
5092 match(RegD);
5093 op_cost(0);
5094 format %{ %}
5095 interface(REG_INTER);
5096 %}
5097
5098 operand vRegD_V4()
5099 %{
5100 constraint(ALLOC_IN_RC(v4_reg));
5101 match(RegD);
5102 op_cost(0);
5103 format %{ %}
5104 interface(REG_INTER);
5105 %}
5106
5107 operand vRegD_V5()
5108 %{
5109 constraint(ALLOC_IN_RC(v5_reg));
5110 match(RegD);
5111 op_cost(0);
5112 format %{ %}
5113 interface(REG_INTER);
5114 %}
5115
5116 operand vRegD_V6()
5117 %{
5118 constraint(ALLOC_IN_RC(v6_reg));
5119 match(RegD);
5120 op_cost(0);
5121 format %{ %}
5122 interface(REG_INTER);
5123 %}
5124
5125 operand vRegD_V7()
5126 %{
5127 constraint(ALLOC_IN_RC(v7_reg));
5128 match(RegD);
5129 op_cost(0);
5130 format %{ %}
5131 interface(REG_INTER);
5132 %}
5133
5134 operand vRegD_V12()
5135 %{
5136 constraint(ALLOC_IN_RC(v12_reg));
5137 match(RegD);
5138 op_cost(0);
5139 format %{ %}
5140 interface(REG_INTER);
5141 %}
5142
5143 operand vRegD_V13()
5144 %{
5145 constraint(ALLOC_IN_RC(v13_reg));
5146 match(RegD);
5147 op_cost(0);
5148 format %{ %}
5149 interface(REG_INTER);
5150 %}
5151
5152 operand pReg()
5153 %{
5154 constraint(ALLOC_IN_RC(pr_reg));
5155 match(RegVectMask);
5156 match(pRegGov);
5157 op_cost(0);
5158 format %{ %}
5159 interface(REG_INTER);
5160 %}
5161
5162 operand pRegGov()
5163 %{
5164 constraint(ALLOC_IN_RC(gov_pr));
5165 match(RegVectMask);
5166 match(pReg);
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 operand pRegGov_P0()
5173 %{
5174 constraint(ALLOC_IN_RC(p0_reg));
5175 match(RegVectMask);
5176 op_cost(0);
5177 format %{ %}
5178 interface(REG_INTER);
5179 %}
5180
5181 operand pRegGov_P1()
5182 %{
5183 constraint(ALLOC_IN_RC(p1_reg));
5184 match(RegVectMask);
5185 op_cost(0);
5186 format %{ %}
5187 interface(REG_INTER);
5188 %}
5189
5190 // Flags register, used as output of signed compare instructions
5191
5192 // note that on AArch64 we also use this register as the output for
5193 // for floating point compare instructions (CmpF CmpD). this ensures
5194 // that ordered inequality tests use GT, GE, LT or LE none of which
5195 // pass through cases where the result is unordered i.e. one or both
5196 // inputs to the compare is a NaN. this means that the ideal code can
5197 // replace e.g. a GT with an LE and not end up capturing the NaN case
5198 // (where the comparison should always fail). EQ and NE tests are
5199 // always generated in ideal code so that unordered folds into the NE
5200 // case, matching the behaviour of AArch64 NE.
5201 //
5202 // This differs from x86 where the outputs of FP compares use a
5203 // special FP flags registers and where compares based on this
5204 // register are distinguished into ordered inequalities (cmpOpUCF) and
5205 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5206 // to explicitly handle the unordered case in branches. x86 also has
5207 // to include extra CMoveX rules to accept a cmpOpUCF input.
5208
5209 operand rFlagsReg()
5210 %{
5211 constraint(ALLOC_IN_RC(int_flags));
5212 match(RegFlags);
5213
5214 op_cost(0);
5215 format %{ "RFLAGS" %}
5216 interface(REG_INTER);
5217 %}
5218
5219 // Flags register, used as output of unsigned compare instructions
5220 operand rFlagsRegU()
5221 %{
5222 constraint(ALLOC_IN_RC(int_flags));
5223 match(RegFlags);
5224
5225 op_cost(0);
5226 format %{ "RFLAGSU" %}
5227 interface(REG_INTER);
5228 %}
5229
5230 // Special Registers
5231
5232 // Method Register
5233 operand inline_cache_RegP(iRegP reg)
5234 %{
5235 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5236 match(reg);
5237 match(iRegPNoSp);
5238 op_cost(0);
5239 format %{ %}
5240 interface(REG_INTER);
5241 %}
5242
5243 // Thread Register
5244 operand thread_RegP(iRegP reg)
5245 %{
5246 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5247 match(reg);
5248 op_cost(0);
5249 format %{ %}
5250 interface(REG_INTER);
5251 %}
5252
5253 //----------Memory Operands----------------------------------------------------
5254
5255 operand indirect(iRegP reg)
5256 %{
5257 constraint(ALLOC_IN_RC(ptr_reg));
5258 match(reg);
5259 op_cost(0);
5260 format %{ "[$reg]" %}
5261 interface(MEMORY_INTER) %{
5262 base($reg);
5263 index(0xffffffff);
5264 scale(0x0);
5265 disp(0x0);
5266 %}
5267 %}
5268
5269 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5270 %{
5271 constraint(ALLOC_IN_RC(ptr_reg));
5272 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5273 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5274 op_cost(0);
5275 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5276 interface(MEMORY_INTER) %{
5277 base($reg);
5278 index($ireg);
5279 scale($scale);
5280 disp(0x0);
5281 %}
5282 %}
5283
5284 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5285 %{
5286 constraint(ALLOC_IN_RC(ptr_reg));
5287 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5288 match(AddP reg (LShiftL lreg scale));
5289 op_cost(0);
5290 format %{ "$reg, $lreg lsl($scale)" %}
5291 interface(MEMORY_INTER) %{
5292 base($reg);
5293 index($lreg);
5294 scale($scale);
5295 disp(0x0);
5296 %}
5297 %}
5298
5299 operand indIndexI2L(iRegP reg, iRegI ireg)
5300 %{
5301 constraint(ALLOC_IN_RC(ptr_reg));
5302 match(AddP reg (ConvI2L ireg));
5303 op_cost(0);
5304 format %{ "$reg, $ireg, 0, I2L" %}
5305 interface(MEMORY_INTER) %{
5306 base($reg);
5307 index($ireg);
5308 scale(0x0);
5309 disp(0x0);
5310 %}
5311 %}
5312
5313 operand indIndex(iRegP reg, iRegL lreg)
5314 %{
5315 constraint(ALLOC_IN_RC(ptr_reg));
5316 match(AddP reg lreg);
5317 op_cost(0);
5318 format %{ "$reg, $lreg" %}
5319 interface(MEMORY_INTER) %{
5320 base($reg);
5321 index($lreg);
5322 scale(0x0);
5323 disp(0x0);
5324 %}
5325 %}
5326
5327 operand indOffI1(iRegP reg, immIOffset1 off)
5328 %{
5329 constraint(ALLOC_IN_RC(ptr_reg));
5330 match(AddP reg off);
5331 op_cost(0);
5332 format %{ "[$reg, $off]" %}
5333 interface(MEMORY_INTER) %{
5334 base($reg);
5335 index(0xffffffff);
5336 scale(0x0);
5337 disp($off);
5338 %}
5339 %}
5340
5341 operand indOffI2(iRegP reg, immIOffset2 off)
5342 %{
5343 constraint(ALLOC_IN_RC(ptr_reg));
5344 match(AddP reg off);
5345 op_cost(0);
5346 format %{ "[$reg, $off]" %}
5347 interface(MEMORY_INTER) %{
5348 base($reg);
5349 index(0xffffffff);
5350 scale(0x0);
5351 disp($off);
5352 %}
5353 %}
5354
5355 operand indOffI4(iRegP reg, immIOffset4 off)
5356 %{
5357 constraint(ALLOC_IN_RC(ptr_reg));
5358 match(AddP reg off);
5359 op_cost(0);
5360 format %{ "[$reg, $off]" %}
5361 interface(MEMORY_INTER) %{
5362 base($reg);
5363 index(0xffffffff);
5364 scale(0x0);
5365 disp($off);
5366 %}
5367 %}
5368
5369 operand indOffI8(iRegP reg, immIOffset8 off)
5370 %{
5371 constraint(ALLOC_IN_RC(ptr_reg));
5372 match(AddP reg off);
5373 op_cost(0);
5374 format %{ "[$reg, $off]" %}
5375 interface(MEMORY_INTER) %{
5376 base($reg);
5377 index(0xffffffff);
5378 scale(0x0);
5379 disp($off);
5380 %}
5381 %}
5382
5383 operand indOffI16(iRegP reg, immIOffset16 off)
5384 %{
5385 constraint(ALLOC_IN_RC(ptr_reg));
5386 match(AddP reg off);
5387 op_cost(0);
5388 format %{ "[$reg, $off]" %}
5389 interface(MEMORY_INTER) %{
5390 base($reg);
5391 index(0xffffffff);
5392 scale(0x0);
5393 disp($off);
5394 %}
5395 %}
5396
5397 operand indOffL1(iRegP reg, immLoffset1 off)
5398 %{
5399 constraint(ALLOC_IN_RC(ptr_reg));
5400 match(AddP reg off);
5401 op_cost(0);
5402 format %{ "[$reg, $off]" %}
5403 interface(MEMORY_INTER) %{
5404 base($reg);
5405 index(0xffffffff);
5406 scale(0x0);
5407 disp($off);
5408 %}
5409 %}
5410
5411 operand indOffL2(iRegP reg, immLoffset2 off)
5412 %{
5413 constraint(ALLOC_IN_RC(ptr_reg));
5414 match(AddP reg off);
5415 op_cost(0);
5416 format %{ "[$reg, $off]" %}
5417 interface(MEMORY_INTER) %{
5418 base($reg);
5419 index(0xffffffff);
5420 scale(0x0);
5421 disp($off);
5422 %}
5423 %}
5424
5425 operand indOffL4(iRegP reg, immLoffset4 off)
5426 %{
5427 constraint(ALLOC_IN_RC(ptr_reg));
5428 match(AddP reg off);
5429 op_cost(0);
5430 format %{ "[$reg, $off]" %}
5431 interface(MEMORY_INTER) %{
5432 base($reg);
5433 index(0xffffffff);
5434 scale(0x0);
5435 disp($off);
5436 %}
5437 %}
5438
5439 operand indOffL8(iRegP reg, immLoffset8 off)
5440 %{
5441 constraint(ALLOC_IN_RC(ptr_reg));
5442 match(AddP reg off);
5443 op_cost(0);
5444 format %{ "[$reg, $off]" %}
5445 interface(MEMORY_INTER) %{
5446 base($reg);
5447 index(0xffffffff);
5448 scale(0x0);
5449 disp($off);
5450 %}
5451 %}
5452
5453 operand indOffL16(iRegP reg, immLoffset16 off)
5454 %{
5455 constraint(ALLOC_IN_RC(ptr_reg));
5456 match(AddP reg off);
5457 op_cost(0);
5458 format %{ "[$reg, $off]" %}
5459 interface(MEMORY_INTER) %{
5460 base($reg);
5461 index(0xffffffff);
5462 scale(0x0);
5463 disp($off);
5464 %}
5465 %}
5466
5467 operand indirectX2P(iRegL reg)
5468 %{
5469 constraint(ALLOC_IN_RC(ptr_reg));
5470 match(CastX2P reg);
5471 op_cost(0);
5472 format %{ "[$reg]\t# long -> ptr" %}
5473 interface(MEMORY_INTER) %{
5474 base($reg);
5475 index(0xffffffff);
5476 scale(0x0);
5477 disp(0x0);
5478 %}
5479 %}
5480
5481 operand indOffX2P(iRegL reg, immLOffset off)
5482 %{
5483 constraint(ALLOC_IN_RC(ptr_reg));
5484 match(AddP (CastX2P reg) off);
5485 op_cost(0);
5486 format %{ "[$reg, $off]\t# long -> ptr" %}
5487 interface(MEMORY_INTER) %{
5488 base($reg);
5489 index(0xffffffff);
5490 scale(0x0);
5491 disp($off);
5492 %}
5493 %}
5494
5495 operand indirectN(iRegN reg)
5496 %{
5497 predicate(CompressedOops::shift() == 0);
5498 constraint(ALLOC_IN_RC(ptr_reg));
5499 match(DecodeN reg);
5500 op_cost(0);
5501 format %{ "[$reg]\t# narrow" %}
5502 interface(MEMORY_INTER) %{
5503 base($reg);
5504 index(0xffffffff);
5505 scale(0x0);
5506 disp(0x0);
5507 %}
5508 %}
5509
5510 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5511 %{
5512 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5513 constraint(ALLOC_IN_RC(ptr_reg));
5514 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5515 op_cost(0);
5516 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5517 interface(MEMORY_INTER) %{
5518 base($reg);
5519 index($ireg);
5520 scale($scale);
5521 disp(0x0);
5522 %}
5523 %}
5524
5525 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5526 %{
5527 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5528 constraint(ALLOC_IN_RC(ptr_reg));
5529 match(AddP (DecodeN reg) (LShiftL lreg scale));
5530 op_cost(0);
5531 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5532 interface(MEMORY_INTER) %{
5533 base($reg);
5534 index($lreg);
5535 scale($scale);
5536 disp(0x0);
5537 %}
5538 %}
5539
5540 operand indIndexI2LN(iRegN reg, iRegI ireg)
5541 %{
5542 predicate(CompressedOops::shift() == 0);
5543 constraint(ALLOC_IN_RC(ptr_reg));
5544 match(AddP (DecodeN reg) (ConvI2L ireg));
5545 op_cost(0);
5546 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5547 interface(MEMORY_INTER) %{
5548 base($reg);
5549 index($ireg);
5550 scale(0x0);
5551 disp(0x0);
5552 %}
5553 %}
5554
5555 operand indIndexN(iRegN reg, iRegL lreg)
5556 %{
5557 predicate(CompressedOops::shift() == 0);
5558 constraint(ALLOC_IN_RC(ptr_reg));
5559 match(AddP (DecodeN reg) lreg);
5560 op_cost(0);
5561 format %{ "$reg, $lreg\t# narrow" %}
5562 interface(MEMORY_INTER) %{
5563 base($reg);
5564 index($lreg);
5565 scale(0x0);
5566 disp(0x0);
5567 %}
5568 %}
5569
5570 operand indOffIN(iRegN reg, immIOffset off)
5571 %{
5572 predicate(CompressedOops::shift() == 0);
5573 constraint(ALLOC_IN_RC(ptr_reg));
5574 match(AddP (DecodeN reg) off);
5575 op_cost(0);
5576 format %{ "[$reg, $off]\t# narrow" %}
5577 interface(MEMORY_INTER) %{
5578 base($reg);
5579 index(0xffffffff);
5580 scale(0x0);
5581 disp($off);
5582 %}
5583 %}
5584
5585 operand indOffLN(iRegN reg, immLOffset off)
5586 %{
5587 predicate(CompressedOops::shift() == 0);
5588 constraint(ALLOC_IN_RC(ptr_reg));
5589 match(AddP (DecodeN reg) off);
5590 op_cost(0);
5591 format %{ "[$reg, $off]\t# narrow" %}
5592 interface(MEMORY_INTER) %{
5593 base($reg);
5594 index(0xffffffff);
5595 scale(0x0);
5596 disp($off);
5597 %}
5598 %}
5599
5600
5601 //----------Special Memory Operands--------------------------------------------
5602 // Stack Slot Operand - This operand is used for loading and storing temporary
5603 // values on the stack where a match requires a value to
5604 // flow through memory.
5605 operand stackSlotP(sRegP reg)
5606 %{
5607 constraint(ALLOC_IN_RC(stack_slots));
5608 op_cost(100);
5609 // No match rule because this operand is only generated in matching
5610 // match(RegP);
5611 format %{ "[$reg]" %}
5612 interface(MEMORY_INTER) %{
5613 base(0x1e); // RSP
5614 index(0x0); // No Index
5615 scale(0x0); // No Scale
5616 disp($reg); // Stack Offset
5617 %}
5618 %}
5619
5620 operand stackSlotI(sRegI reg)
5621 %{
5622 constraint(ALLOC_IN_RC(stack_slots));
5623 // No match rule because this operand is only generated in matching
5624 // match(RegI);
5625 format %{ "[$reg]" %}
5626 interface(MEMORY_INTER) %{
5627 base(0x1e); // RSP
5628 index(0x0); // No Index
5629 scale(0x0); // No Scale
5630 disp($reg); // Stack Offset
5631 %}
5632 %}
5633
5634 operand stackSlotF(sRegF reg)
5635 %{
5636 constraint(ALLOC_IN_RC(stack_slots));
5637 // No match rule because this operand is only generated in matching
5638 // match(RegF);
5639 format %{ "[$reg]" %}
5640 interface(MEMORY_INTER) %{
5641 base(0x1e); // RSP
5642 index(0x0); // No Index
5643 scale(0x0); // No Scale
5644 disp($reg); // Stack Offset
5645 %}
5646 %}
5647
5648 operand stackSlotD(sRegD reg)
5649 %{
5650 constraint(ALLOC_IN_RC(stack_slots));
5651 // No match rule because this operand is only generated in matching
5652 // match(RegD);
5653 format %{ "[$reg]" %}
5654 interface(MEMORY_INTER) %{
5655 base(0x1e); // RSP
5656 index(0x0); // No Index
5657 scale(0x0); // No Scale
5658 disp($reg); // Stack Offset
5659 %}
5660 %}
5661
5662 operand stackSlotL(sRegL reg)
5663 %{
5664 constraint(ALLOC_IN_RC(stack_slots));
5665 // No match rule because this operand is only generated in matching
5666 // match(RegL);
5667 format %{ "[$reg]" %}
5668 interface(MEMORY_INTER) %{
5669 base(0x1e); // RSP
5670 index(0x0); // No Index
5671 scale(0x0); // No Scale
5672 disp($reg); // Stack Offset
5673 %}
5674 %}
5675
5676 // Operands for expressing Control Flow
5677 // NOTE: Label is a predefined operand which should not be redefined in
5678 // the AD file. It is generically handled within the ADLC.
5679
5680 //----------Conditional Branch Operands----------------------------------------
5681 // Comparison Op - This is the operation of the comparison, and is limited to
5682 // the following set of codes:
5683 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5684 //
5685 // Other attributes of the comparison, such as unsignedness, are specified
5686 // by the comparison instruction that sets a condition code flags register.
5687 // That result is represented by a flags operand whose subtype is appropriate
5688 // to the unsignedness (etc.) of the comparison.
5689 //
5690 // Later, the instruction which matches both the Comparison Op (a Bool) and
5691 // the flags (produced by the Cmp) specifies the coding of the comparison op
5692 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5693
5694 // used for signed integral comparisons and fp comparisons
5695
5696 operand cmpOp()
5697 %{
5698 match(Bool);
5699
5700 format %{ "" %}
5701 interface(COND_INTER) %{
5702 equal(0x0, "eq");
5703 not_equal(0x1, "ne");
5704 less(0xb, "lt");
5705 greater_equal(0xa, "ge");
5706 less_equal(0xd, "le");
5707 greater(0xc, "gt");
5708 overflow(0x6, "vs");
5709 no_overflow(0x7, "vc");
5710 %}
5711 %}
5712
5713 // used for unsigned integral comparisons
5714
5715 operand cmpOpU()
5716 %{
5717 match(Bool);
5718
5719 format %{ "" %}
5720 interface(COND_INTER) %{
5721 equal(0x0, "eq");
5722 not_equal(0x1, "ne");
5723 less(0x3, "lo");
5724 greater_equal(0x2, "hs");
5725 less_equal(0x9, "ls");
5726 greater(0x8, "hi");
5727 overflow(0x6, "vs");
5728 no_overflow(0x7, "vc");
5729 %}
5730 %}
5731
5732 // used for certain integral comparisons which can be
5733 // converted to cbxx or tbxx instructions
5734
5735 operand cmpOpEqNe()
5736 %{
5737 match(Bool);
5738 op_cost(0);
5739 predicate(n->as_Bool()->_test._test == BoolTest::ne
5740 || n->as_Bool()->_test._test == BoolTest::eq);
5741
5742 format %{ "" %}
5743 interface(COND_INTER) %{
5744 equal(0x0, "eq");
5745 not_equal(0x1, "ne");
5746 less(0xb, "lt");
5747 greater_equal(0xa, "ge");
5748 less_equal(0xd, "le");
5749 greater(0xc, "gt");
5750 overflow(0x6, "vs");
5751 no_overflow(0x7, "vc");
5752 %}
5753 %}
5754
5755 // used for certain integral comparisons which can be
5756 // converted to cbxx or tbxx instructions
5757
5758 operand cmpOpLtGe()
5759 %{
5760 match(Bool);
5761 op_cost(0);
5762
5763 predicate(n->as_Bool()->_test._test == BoolTest::lt
5764 || n->as_Bool()->_test._test == BoolTest::ge);
5765
5766 format %{ "" %}
5767 interface(COND_INTER) %{
5768 equal(0x0, "eq");
5769 not_equal(0x1, "ne");
5770 less(0xb, "lt");
5771 greater_equal(0xa, "ge");
5772 less_equal(0xd, "le");
5773 greater(0xc, "gt");
5774 overflow(0x6, "vs");
5775 no_overflow(0x7, "vc");
5776 %}
5777 %}
5778
5779 // used for certain unsigned integral comparisons which can be
5780 // converted to cbxx or tbxx instructions
5781
5782 operand cmpOpUEqNeLeGt()
5783 %{
5784 match(Bool);
5785 op_cost(0);
5786
5787 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5788 n->as_Bool()->_test._test == BoolTest::ne ||
5789 n->as_Bool()->_test._test == BoolTest::le ||
5790 n->as_Bool()->_test._test == BoolTest::gt);
5791
5792 format %{ "" %}
5793 interface(COND_INTER) %{
5794 equal(0x0, "eq");
5795 not_equal(0x1, "ne");
5796 less(0x3, "lo");
5797 greater_equal(0x2, "hs");
5798 less_equal(0x9, "ls");
5799 greater(0x8, "hi");
5800 overflow(0x6, "vs");
5801 no_overflow(0x7, "vc");
5802 %}
5803 %}
5804
5805 // Special operand allowing long args to int ops to be truncated for free
5806
5807 operand iRegL2I(iRegL reg) %{
5808
5809 op_cost(0);
5810
5811 match(ConvL2I reg);
5812
5813 format %{ "l2i($reg)" %}
5814
5815 interface(REG_INTER)
5816 %}
5817
5818 operand iRegL2P(iRegL reg) %{
5819
5820 op_cost(0);
5821
5822 match(CastX2P reg);
5823
5824 format %{ "l2p($reg)" %}
5825
5826 interface(REG_INTER)
5827 %}
5828
5829 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5830 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5831 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5832 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5833
5834 //----------OPERAND CLASSES----------------------------------------------------
5835 // Operand Classes are groups of operands that are used as to simplify
5836 // instruction definitions by not requiring the AD writer to specify
5837 // separate instructions for every form of operand when the
5838 // instruction accepts multiple operand types with the same basic
5839 // encoding and format. The classic case of this is memory operands.
5840
5841 // memory is used to define read/write location for load/store
5842 // instruction defs. we can turn a memory op into an Address
5843
5844 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5845 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5846
5847 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5848 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5849
5850 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5851 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5852
5853 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5854 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5855
5856 // All of the memory operands. For the pipeline description.
5857 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5858 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5859 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5860
5861
5862 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5863 // operations. it allows the src to be either an iRegI or a (ConvL2I
5864 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5865 // can be elided because the 32-bit instruction will just employ the
5866 // lower 32 bits anyway.
5867 //
5868 // n.b. this does not elide all L2I conversions. if the truncated
5869 // value is consumed by more than one operation then the ConvL2I
5870 // cannot be bundled into the consuming nodes so an l2i gets planted
5871 // (actually a movw $dst $src) and the downstream instructions consume
5872 // the result of the l2i as an iRegI input. That's a shame since the
5873 // movw is actually redundant but its not too costly.
5874
5875 opclass iRegIorL2I(iRegI, iRegL2I);
5876 opclass iRegPorL2P(iRegP, iRegL2P);
5877
5878 //----------PIPELINE-----------------------------------------------------------
5879 // Rules which define the behavior of the target architectures pipeline.
5880
5881 // For specific pipelines, eg A53, define the stages of that pipeline
5882 //pipe_desc(ISS, EX1, EX2, WR);
5883 #define ISS S0
5884 #define EX1 S1
5885 #define EX2 S2
5886 #define WR S3
5887
5888 // Integer ALU reg operation
5889 pipeline %{
5890
5891 attributes %{
5892 // ARM instructions are of fixed length
5893 fixed_size_instructions; // Fixed size instructions TODO does
5894 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5895 // ARM instructions come in 32-bit word units
5896 instruction_unit_size = 4; // An instruction is 4 bytes long
5897 instruction_fetch_unit_size = 64; // The processor fetches one line
5898 instruction_fetch_units = 1; // of 64 bytes
5899 %}
5900
5901 // We don't use an actual pipeline model so don't care about resources
5902 // or description. we do use pipeline classes to introduce fixed
5903 // latencies
5904
5905 //----------RESOURCES----------------------------------------------------------
5906 // Resources are the functional units available to the machine
5907
5908 resources( INS0, INS1, INS01 = INS0 | INS1,
5909 ALU0, ALU1, ALU = ALU0 | ALU1,
5910 MAC,
5911 DIV,
5912 BRANCH,
5913 LDST,
5914 NEON_FP);
5915
5916 //----------PIPELINE DESCRIPTION-----------------------------------------------
5917 // Pipeline Description specifies the stages in the machine's pipeline
5918
5919 // Define the pipeline as a generic 6 stage pipeline
5920 pipe_desc(S0, S1, S2, S3, S4, S5);
5921
5922 //----------PIPELINE CLASSES---------------------------------------------------
5923 // Pipeline Classes describe the stages in which input and output are
5924 // referenced by the hardware pipeline.
5925
5926 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5927 %{
5928 single_instruction;
5929 src1 : S1(read);
5930 src2 : S2(read);
5931 dst : S5(write);
5932 INS01 : ISS;
5933 NEON_FP : S5;
5934 %}
5935
5936 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5937 %{
5938 single_instruction;
5939 src1 : S1(read);
5940 src2 : S2(read);
5941 dst : S5(write);
5942 INS01 : ISS;
5943 NEON_FP : S5;
5944 %}
5945
5946 pipe_class fp_uop_s(vRegF dst, vRegF src)
5947 %{
5948 single_instruction;
5949 src : S1(read);
5950 dst : S5(write);
5951 INS01 : ISS;
5952 NEON_FP : S5;
5953 %}
5954
5955 pipe_class fp_uop_d(vRegD dst, vRegD src)
5956 %{
5957 single_instruction;
5958 src : S1(read);
5959 dst : S5(write);
5960 INS01 : ISS;
5961 NEON_FP : S5;
5962 %}
5963
5964 pipe_class fp_d2f(vRegF dst, vRegD src)
5965 %{
5966 single_instruction;
5967 src : S1(read);
5968 dst : S5(write);
5969 INS01 : ISS;
5970 NEON_FP : S5;
5971 %}
5972
5973 pipe_class fp_f2d(vRegD dst, vRegF src)
5974 %{
5975 single_instruction;
5976 src : S1(read);
5977 dst : S5(write);
5978 INS01 : ISS;
5979 NEON_FP : S5;
5980 %}
5981
5982 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5983 %{
5984 single_instruction;
5985 src : S1(read);
5986 dst : S5(write);
5987 INS01 : ISS;
5988 NEON_FP : S5;
5989 %}
5990
5991 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5992 %{
5993 single_instruction;
5994 src : S1(read);
5995 dst : S5(write);
5996 INS01 : ISS;
5997 NEON_FP : S5;
5998 %}
5999
6000 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6001 %{
6002 single_instruction;
6003 src : S1(read);
6004 dst : S5(write);
6005 INS01 : ISS;
6006 NEON_FP : S5;
6007 %}
6008
6009 pipe_class fp_l2f(vRegF dst, iRegL src)
6010 %{
6011 single_instruction;
6012 src : S1(read);
6013 dst : S5(write);
6014 INS01 : ISS;
6015 NEON_FP : S5;
6016 %}
6017
6018 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6019 %{
6020 single_instruction;
6021 src : S1(read);
6022 dst : S5(write);
6023 INS01 : ISS;
6024 NEON_FP : S5;
6025 %}
6026
6027 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6028 %{
6029 single_instruction;
6030 src : S1(read);
6031 dst : S5(write);
6032 INS01 : ISS;
6033 NEON_FP : S5;
6034 %}
6035
6036 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6037 %{
6038 single_instruction;
6039 src : S1(read);
6040 dst : S5(write);
6041 INS01 : ISS;
6042 NEON_FP : S5;
6043 %}
6044
6045 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6046 %{
6047 single_instruction;
6048 src : S1(read);
6049 dst : S5(write);
6050 INS01 : ISS;
6051 NEON_FP : S5;
6052 %}
6053
6054 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6055 %{
6056 single_instruction;
6057 src1 : S1(read);
6058 src2 : S2(read);
6059 dst : S5(write);
6060 INS0 : ISS;
6061 NEON_FP : S5;
6062 %}
6063
6064 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6065 %{
6066 single_instruction;
6067 src1 : S1(read);
6068 src2 : S2(read);
6069 dst : S5(write);
6070 INS0 : ISS;
6071 NEON_FP : S5;
6072 %}
6073
6074 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6075 %{
6076 single_instruction;
6077 cr : S1(read);
6078 src1 : S1(read);
6079 src2 : S1(read);
6080 dst : S3(write);
6081 INS01 : ISS;
6082 NEON_FP : S3;
6083 %}
6084
6085 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6086 %{
6087 single_instruction;
6088 cr : S1(read);
6089 src1 : S1(read);
6090 src2 : S1(read);
6091 dst : S3(write);
6092 INS01 : ISS;
6093 NEON_FP : S3;
6094 %}
6095
6096 pipe_class fp_imm_s(vRegF dst)
6097 %{
6098 single_instruction;
6099 dst : S3(write);
6100 INS01 : ISS;
6101 NEON_FP : S3;
6102 %}
6103
6104 pipe_class fp_imm_d(vRegD dst)
6105 %{
6106 single_instruction;
6107 dst : S3(write);
6108 INS01 : ISS;
6109 NEON_FP : S3;
6110 %}
6111
6112 pipe_class fp_load_constant_s(vRegF dst)
6113 %{
6114 single_instruction;
6115 dst : S4(write);
6116 INS01 : ISS;
6117 NEON_FP : S4;
6118 %}
6119
6120 pipe_class fp_load_constant_d(vRegD dst)
6121 %{
6122 single_instruction;
6123 dst : S4(write);
6124 INS01 : ISS;
6125 NEON_FP : S4;
6126 %}
6127
6128 //------- Integer ALU operations --------------------------
6129
6130 // Integer ALU reg-reg operation
6131 // Operands needed in EX1, result generated in EX2
6132 // Eg. ADD x0, x1, x2
6133 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6134 %{
6135 single_instruction;
6136 dst : EX2(write);
6137 src1 : EX1(read);
6138 src2 : EX1(read);
6139 INS01 : ISS; // Dual issue as instruction 0 or 1
6140 ALU : EX2;
6141 %}
6142
6143 // Integer ALU reg-reg operation with constant shift
6144 // Shifted register must be available in LATE_ISS instead of EX1
6145 // Eg. ADD x0, x1, x2, LSL #2
6146 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6147 %{
6148 single_instruction;
6149 dst : EX2(write);
6150 src1 : EX1(read);
6151 src2 : ISS(read);
6152 INS01 : ISS;
6153 ALU : EX2;
6154 %}
6155
6156 // Integer ALU reg operation with constant shift
6157 // Eg. LSL x0, x1, #shift
6158 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6159 %{
6160 single_instruction;
6161 dst : EX2(write);
6162 src1 : ISS(read);
6163 INS01 : ISS;
6164 ALU : EX2;
6165 %}
6166
6167 // Integer ALU reg-reg operation with variable shift
6168 // Both operands must be available in LATE_ISS instead of EX1
6169 // Result is available in EX1 instead of EX2
6170 // Eg. LSLV x0, x1, x2
6171 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6172 %{
6173 single_instruction;
6174 dst : EX1(write);
6175 src1 : ISS(read);
6176 src2 : ISS(read);
6177 INS01 : ISS;
6178 ALU : EX1;
6179 %}
6180
6181 // Integer ALU reg-reg operation with extract
6182 // As for _vshift above, but result generated in EX2
6183 // Eg. EXTR x0, x1, x2, #N
6184 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6185 %{
6186 single_instruction;
6187 dst : EX2(write);
6188 src1 : ISS(read);
6189 src2 : ISS(read);
6190 INS1 : ISS; // Can only dual issue as Instruction 1
6191 ALU : EX1;
6192 %}
6193
6194 // Integer ALU reg operation
6195 // Eg. NEG x0, x1
6196 pipe_class ialu_reg(iRegI dst, iRegI src)
6197 %{
6198 single_instruction;
6199 dst : EX2(write);
6200 src : EX1(read);
6201 INS01 : ISS;
6202 ALU : EX2;
6203 %}
6204
6205 // Integer ALU reg mmediate operation
6206 // Eg. ADD x0, x1, #N
6207 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6208 %{
6209 single_instruction;
6210 dst : EX2(write);
6211 src1 : EX1(read);
6212 INS01 : ISS;
6213 ALU : EX2;
6214 %}
6215
6216 // Integer ALU immediate operation (no source operands)
6217 // Eg. MOV x0, #N
6218 pipe_class ialu_imm(iRegI dst)
6219 %{
6220 single_instruction;
6221 dst : EX1(write);
6222 INS01 : ISS;
6223 ALU : EX1;
6224 %}
6225
6226 //------- Compare operation -------------------------------
6227
6228 // Compare reg-reg
6229 // Eg. CMP x0, x1
6230 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6231 %{
6232 single_instruction;
6233 // fixed_latency(16);
6234 cr : EX2(write);
6235 op1 : EX1(read);
6236 op2 : EX1(read);
6237 INS01 : ISS;
6238 ALU : EX2;
6239 %}
6240
6241 // Compare reg-reg
6242 // Eg. CMP x0, #N
6243 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6244 %{
6245 single_instruction;
6246 // fixed_latency(16);
6247 cr : EX2(write);
6248 op1 : EX1(read);
6249 INS01 : ISS;
6250 ALU : EX2;
6251 %}
6252
6253 //------- Conditional instructions ------------------------
6254
6255 // Conditional no operands
6256 // Eg. CSINC x0, zr, zr, <cond>
6257 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6258 %{
6259 single_instruction;
6260 cr : EX1(read);
6261 dst : EX2(write);
6262 INS01 : ISS;
6263 ALU : EX2;
6264 %}
6265
6266 // Conditional 2 operand
6267 // EG. CSEL X0, X1, X2, <cond>
6268 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6269 %{
6270 single_instruction;
6271 cr : EX1(read);
6272 src1 : EX1(read);
6273 src2 : EX1(read);
6274 dst : EX2(write);
6275 INS01 : ISS;
6276 ALU : EX2;
6277 %}
6278
6279 // Conditional 2 operand
6280 // EG. CSEL X0, X1, X2, <cond>
6281 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6282 %{
6283 single_instruction;
6284 cr : EX1(read);
6285 src : EX1(read);
6286 dst : EX2(write);
6287 INS01 : ISS;
6288 ALU : EX2;
6289 %}
6290
6291 //------- Multiply pipeline operations --------------------
6292
6293 // Multiply reg-reg
6294 // Eg. MUL w0, w1, w2
6295 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6296 %{
6297 single_instruction;
6298 dst : WR(write);
6299 src1 : ISS(read);
6300 src2 : ISS(read);
6301 INS01 : ISS;
6302 MAC : WR;
6303 %}
6304
6305 // Multiply accumulate
6306 // Eg. MADD w0, w1, w2, w3
6307 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6308 %{
6309 single_instruction;
6310 dst : WR(write);
6311 src1 : ISS(read);
6312 src2 : ISS(read);
6313 src3 : ISS(read);
6314 INS01 : ISS;
6315 MAC : WR;
6316 %}
6317
6318 // Eg. MUL w0, w1, w2
6319 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6320 %{
6321 single_instruction;
6322 fixed_latency(3); // Maximum latency for 64 bit mul
6323 dst : WR(write);
6324 src1 : ISS(read);
6325 src2 : ISS(read);
6326 INS01 : ISS;
6327 MAC : WR;
6328 %}
6329
6330 // Multiply accumulate
6331 // Eg. MADD w0, w1, w2, w3
6332 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6333 %{
6334 single_instruction;
6335 fixed_latency(3); // Maximum latency for 64 bit mul
6336 dst : WR(write);
6337 src1 : ISS(read);
6338 src2 : ISS(read);
6339 src3 : ISS(read);
6340 INS01 : ISS;
6341 MAC : WR;
6342 %}
6343
6344 //------- Divide pipeline operations --------------------
6345
6346 // Eg. SDIV w0, w1, w2
6347 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6348 %{
6349 single_instruction;
6350 fixed_latency(8); // Maximum latency for 32 bit divide
6351 dst : WR(write);
6352 src1 : ISS(read);
6353 src2 : ISS(read);
6354 INS0 : ISS; // Can only dual issue as instruction 0
6355 DIV : WR;
6356 %}
6357
6358 // Eg. SDIV x0, x1, x2
6359 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6360 %{
6361 single_instruction;
6362 fixed_latency(16); // Maximum latency for 64 bit divide
6363 dst : WR(write);
6364 src1 : ISS(read);
6365 src2 : ISS(read);
6366 INS0 : ISS; // Can only dual issue as instruction 0
6367 DIV : WR;
6368 %}
6369
6370 //------- Load pipeline operations ------------------------
6371
6372 // Load - prefetch
6373 // Eg. PFRM <mem>
6374 pipe_class iload_prefetch(memory mem)
6375 %{
6376 single_instruction;
6377 mem : ISS(read);
6378 INS01 : ISS;
6379 LDST : WR;
6380 %}
6381
6382 // Load - reg, mem
6383 // Eg. LDR x0, <mem>
6384 pipe_class iload_reg_mem(iRegI dst, memory mem)
6385 %{
6386 single_instruction;
6387 dst : WR(write);
6388 mem : ISS(read);
6389 INS01 : ISS;
6390 LDST : WR;
6391 %}
6392
6393 // Load - reg, reg
6394 // Eg. LDR x0, [sp, x1]
6395 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6396 %{
6397 single_instruction;
6398 dst : WR(write);
6399 src : ISS(read);
6400 INS01 : ISS;
6401 LDST : WR;
6402 %}
6403
6404 //------- Store pipeline operations -----------------------
6405
6406 // Store - zr, mem
6407 // Eg. STR zr, <mem>
6408 pipe_class istore_mem(memory mem)
6409 %{
6410 single_instruction;
6411 mem : ISS(read);
6412 INS01 : ISS;
6413 LDST : WR;
6414 %}
6415
6416 // Store - reg, mem
6417 // Eg. STR x0, <mem>
6418 pipe_class istore_reg_mem(iRegI src, memory mem)
6419 %{
6420 single_instruction;
6421 mem : ISS(read);
6422 src : EX2(read);
6423 INS01 : ISS;
6424 LDST : WR;
6425 %}
6426
6427 // Store - reg, reg
6428 // Eg. STR x0, [sp, x1]
6429 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6430 %{
6431 single_instruction;
6432 dst : ISS(read);
6433 src : EX2(read);
6434 INS01 : ISS;
6435 LDST : WR;
6436 %}
6437
6438 //------- Store pipeline operations -----------------------
6439
6440 // Branch
6441 pipe_class pipe_branch()
6442 %{
6443 single_instruction;
6444 INS01 : ISS;
6445 BRANCH : EX1;
6446 %}
6447
6448 // Conditional branch
6449 pipe_class pipe_branch_cond(rFlagsReg cr)
6450 %{
6451 single_instruction;
6452 cr : EX1(read);
6453 INS01 : ISS;
6454 BRANCH : EX1;
6455 %}
6456
6457 // Compare & Branch
6458 // EG. CBZ/CBNZ
6459 pipe_class pipe_cmp_branch(iRegI op1)
6460 %{
6461 single_instruction;
6462 op1 : EX1(read);
6463 INS01 : ISS;
6464 BRANCH : EX1;
6465 %}
6466
6467 //------- Synchronisation operations ----------------------
6468
6469 // Any operation requiring serialization.
6470 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6471 pipe_class pipe_serial()
6472 %{
6473 single_instruction;
6474 force_serialization;
6475 fixed_latency(16);
6476 INS01 : ISS(2); // Cannot dual issue with any other instruction
6477 LDST : WR;
6478 %}
6479
6480 // Generic big/slow expanded idiom - also serialized
6481 pipe_class pipe_slow()
6482 %{
6483 instruction_count(10);
6484 multiple_bundles;
6485 force_serialization;
6486 fixed_latency(16);
6487 INS01 : ISS(2); // Cannot dual issue with any other instruction
6488 LDST : WR;
6489 %}
6490
6491 // Empty pipeline class
6492 pipe_class pipe_class_empty()
6493 %{
6494 single_instruction;
6495 fixed_latency(0);
6496 %}
6497
6498 // Default pipeline class.
6499 pipe_class pipe_class_default()
6500 %{
6501 single_instruction;
6502 fixed_latency(2);
6503 %}
6504
6505 // Pipeline class for compares.
6506 pipe_class pipe_class_compare()
6507 %{
6508 single_instruction;
6509 fixed_latency(16);
6510 %}
6511
6512 // Pipeline class for memory operations.
6513 pipe_class pipe_class_memory()
6514 %{
6515 single_instruction;
6516 fixed_latency(16);
6517 %}
6518
6519 // Pipeline class for call.
6520 pipe_class pipe_class_call()
6521 %{
6522 single_instruction;
6523 fixed_latency(100);
6524 %}
6525
6526 // Define the class for the Nop node.
6527 define %{
6528 MachNop = pipe_class_empty;
6529 %}
6530
6531 %}
6532 //----------INSTRUCTIONS-------------------------------------------------------
6533 //
6534 // match -- States which machine-independent subtree may be replaced
6535 // by this instruction.
6536 // ins_cost -- The estimated cost of this instruction is used by instruction
6537 // selection to identify a minimum cost tree of machine
6538 // instructions that matches a tree of machine-independent
6539 // instructions.
6540 // format -- A string providing the disassembly for this instruction.
6541 // The value of an instruction's operand may be inserted
6542 // by referring to it with a '$' prefix.
6543 // opcode -- Three instruction opcodes may be provided. These are referred
6544 // to within an encode class as $primary, $secondary, and $tertiary
6545 // rrspectively. The primary opcode is commonly used to
6546 // indicate the type of machine instruction, while secondary
6547 // and tertiary are often used for prefix options or addressing
6548 // modes.
6549 // ins_encode -- A list of encode classes with parameters. The encode class
6550 // name must have been defined in an 'enc_class' specification
6551 // in the encode section of the architecture description.
6552
6553 // ============================================================================
6554 // Memory (Load/Store) Instructions
6555
6556 // Load Instructions
6557
6558 // Load Byte (8 bit signed)
6559 instruct loadB(iRegINoSp dst, memory1 mem)
6560 %{
6561 match(Set dst (LoadB mem));
6562 predicate(!needs_acquiring_load(n));
6563
6564 ins_cost(4 * INSN_COST);
6565 format %{ "ldrsbw $dst, $mem\t# byte" %}
6566
6567 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6568
6569 ins_pipe(iload_reg_mem);
6570 %}
6571
6572 // Load Byte (8 bit signed) into long
6573 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6574 %{
6575 match(Set dst (ConvI2L (LoadB mem)));
6576 predicate(!needs_acquiring_load(n->in(1)));
6577
6578 ins_cost(4 * INSN_COST);
6579 format %{ "ldrsb $dst, $mem\t# byte" %}
6580
6581 ins_encode(aarch64_enc_ldrsb(dst, mem));
6582
6583 ins_pipe(iload_reg_mem);
6584 %}
6585
6586 // Load Byte (8 bit unsigned)
6587 instruct loadUB(iRegINoSp dst, memory1 mem)
6588 %{
6589 match(Set dst (LoadUB mem));
6590 predicate(!needs_acquiring_load(n));
6591
6592 ins_cost(4 * INSN_COST);
6593 format %{ "ldrbw $dst, $mem\t# byte" %}
6594
6595 ins_encode(aarch64_enc_ldrb(dst, mem));
6596
6597 ins_pipe(iload_reg_mem);
6598 %}
6599
6600 // Load Byte (8 bit unsigned) into long
6601 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6602 %{
6603 match(Set dst (ConvI2L (LoadUB mem)));
6604 predicate(!needs_acquiring_load(n->in(1)));
6605
6606 ins_cost(4 * INSN_COST);
6607 format %{ "ldrb $dst, $mem\t# byte" %}
6608
6609 ins_encode(aarch64_enc_ldrb(dst, mem));
6610
6611 ins_pipe(iload_reg_mem);
6612 %}
6613
6614 // Load Short (16 bit signed)
6615 instruct loadS(iRegINoSp dst, memory2 mem)
6616 %{
6617 match(Set dst (LoadS mem));
6618 predicate(!needs_acquiring_load(n));
6619
6620 ins_cost(4 * INSN_COST);
6621 format %{ "ldrshw $dst, $mem\t# short" %}
6622
6623 ins_encode(aarch64_enc_ldrshw(dst, mem));
6624
6625 ins_pipe(iload_reg_mem);
6626 %}
6627
6628 // Load Short (16 bit signed) into long
6629 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6630 %{
6631 match(Set dst (ConvI2L (LoadS mem)));
6632 predicate(!needs_acquiring_load(n->in(1)));
6633
6634 ins_cost(4 * INSN_COST);
6635 format %{ "ldrsh $dst, $mem\t# short" %}
6636
6637 ins_encode(aarch64_enc_ldrsh(dst, mem));
6638
6639 ins_pipe(iload_reg_mem);
6640 %}
6641
6642 // Load Char (16 bit unsigned)
6643 instruct loadUS(iRegINoSp dst, memory2 mem)
6644 %{
6645 match(Set dst (LoadUS mem));
6646 predicate(!needs_acquiring_load(n));
6647
6648 ins_cost(4 * INSN_COST);
6649 format %{ "ldrh $dst, $mem\t# short" %}
6650
6651 ins_encode(aarch64_enc_ldrh(dst, mem));
6652
6653 ins_pipe(iload_reg_mem);
6654 %}
6655
6656 // Load Short/Char (16 bit unsigned) into long
6657 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6658 %{
6659 match(Set dst (ConvI2L (LoadUS mem)));
6660 predicate(!needs_acquiring_load(n->in(1)));
6661
6662 ins_cost(4 * INSN_COST);
6663 format %{ "ldrh $dst, $mem\t# short" %}
6664
6665 ins_encode(aarch64_enc_ldrh(dst, mem));
6666
6667 ins_pipe(iload_reg_mem);
6668 %}
6669
6670 // Load Integer (32 bit signed)
6671 instruct loadI(iRegINoSp dst, memory4 mem)
6672 %{
6673 match(Set dst (LoadI mem));
6674 predicate(!needs_acquiring_load(n));
6675
6676 ins_cost(4 * INSN_COST);
6677 format %{ "ldrw $dst, $mem\t# int" %}
6678
6679 ins_encode(aarch64_enc_ldrw(dst, mem));
6680
6681 ins_pipe(iload_reg_mem);
6682 %}
6683
6684 // Load Integer (32 bit signed) into long
6685 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6686 %{
6687 match(Set dst (ConvI2L (LoadI mem)));
6688 predicate(!needs_acquiring_load(n->in(1)));
6689
6690 ins_cost(4 * INSN_COST);
6691 format %{ "ldrsw $dst, $mem\t# int" %}
6692
6693 ins_encode(aarch64_enc_ldrsw(dst, mem));
6694
6695 ins_pipe(iload_reg_mem);
6696 %}
6697
6698 // Load Integer (32 bit unsigned) into long
6699 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6700 %{
6701 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6702 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6703
6704 ins_cost(4 * INSN_COST);
6705 format %{ "ldrw $dst, $mem\t# int" %}
6706
6707 ins_encode(aarch64_enc_ldrw(dst, mem));
6708
6709 ins_pipe(iload_reg_mem);
6710 %}
6711
6712 // Load Long (64 bit signed)
6713 instruct loadL(iRegLNoSp dst, memory8 mem)
6714 %{
6715 match(Set dst (LoadL mem));
6716 predicate(!needs_acquiring_load(n));
6717
6718 ins_cost(4 * INSN_COST);
6719 format %{ "ldr $dst, $mem\t# int" %}
6720
6721 ins_encode(aarch64_enc_ldr(dst, mem));
6722
6723 ins_pipe(iload_reg_mem);
6724 %}
6725
6726 // Load Range
6727 instruct loadRange(iRegINoSp dst, memory4 mem)
6728 %{
6729 match(Set dst (LoadRange mem));
6730
6731 ins_cost(4 * INSN_COST);
6732 format %{ "ldrw $dst, $mem\t# range" %}
6733
6734 ins_encode(aarch64_enc_ldrw(dst, mem));
6735
6736 ins_pipe(iload_reg_mem);
6737 %}
6738
6739 // Load Pointer
6740 instruct loadP(iRegPNoSp dst, memory8 mem)
6741 %{
6742 match(Set dst (LoadP mem));
6743 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6744
6745 ins_cost(4 * INSN_COST);
6746 format %{ "ldr $dst, $mem\t# ptr" %}
6747
6748 ins_encode(aarch64_enc_ldr(dst, mem));
6749
6750 ins_pipe(iload_reg_mem);
6751 %}
6752
6753 // Load Compressed Pointer
6754 instruct loadN(iRegNNoSp dst, memory4 mem)
6755 %{
6756 match(Set dst (LoadN mem));
6757 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6758
6759 ins_cost(4 * INSN_COST);
6760 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6761
6762 ins_encode(aarch64_enc_ldrw(dst, mem));
6763
6764 ins_pipe(iload_reg_mem);
6765 %}
6766
6767 // Load Klass Pointer
6768 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6769 %{
6770 match(Set dst (LoadKlass mem));
6771 predicate(!needs_acquiring_load(n));
6772
6773 ins_cost(4 * INSN_COST);
6774 format %{ "ldr $dst, $mem\t# class" %}
6775
6776 ins_encode(aarch64_enc_ldr(dst, mem));
6777
6778 ins_pipe(iload_reg_mem);
6779 %}
6780
6781 // Load Narrow Klass Pointer
6782 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6783 %{
6784 match(Set dst (LoadNKlass mem));
6785 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6786
6787 ins_cost(4 * INSN_COST);
6788 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6789
6790 ins_encode(aarch64_enc_ldrw(dst, mem));
6791
6792 ins_pipe(iload_reg_mem);
6793 %}
6794
6795 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6796 %{
6797 match(Set dst (LoadNKlass mem));
6798 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6799
6800 ins_cost(4 * INSN_COST);
6801 format %{
6802 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6803 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6804 %}
6805 ins_encode %{
6806 // inlined aarch64_enc_ldrw
6807 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6808 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6809 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6810 %}
6811 ins_pipe(iload_reg_mem);
6812 %}
6813
6814 // Load Float
6815 instruct loadF(vRegF dst, memory4 mem)
6816 %{
6817 match(Set dst (LoadF mem));
6818 predicate(!needs_acquiring_load(n));
6819
6820 ins_cost(4 * INSN_COST);
6821 format %{ "ldrs $dst, $mem\t# float" %}
6822
6823 ins_encode( aarch64_enc_ldrs(dst, mem) );
6824
6825 ins_pipe(pipe_class_memory);
6826 %}
6827
6828 // Load Double
6829 instruct loadD(vRegD dst, memory8 mem)
6830 %{
6831 match(Set dst (LoadD mem));
6832 predicate(!needs_acquiring_load(n));
6833
6834 ins_cost(4 * INSN_COST);
6835 format %{ "ldrd $dst, $mem\t# double" %}
6836
6837 ins_encode( aarch64_enc_ldrd(dst, mem) );
6838
6839 ins_pipe(pipe_class_memory);
6840 %}
6841
6842
6843 // Load Int Constant
6844 instruct loadConI(iRegINoSp dst, immI src)
6845 %{
6846 match(Set dst src);
6847
6848 ins_cost(INSN_COST);
6849 format %{ "mov $dst, $src\t# int" %}
6850
6851 ins_encode( aarch64_enc_movw_imm(dst, src) );
6852
6853 ins_pipe(ialu_imm);
6854 %}
6855
6856 // Load Long Constant
6857 instruct loadConL(iRegLNoSp dst, immL src)
6858 %{
6859 match(Set dst src);
6860
6861 ins_cost(INSN_COST);
6862 format %{ "mov $dst, $src\t# long" %}
6863
6864 ins_encode( aarch64_enc_mov_imm(dst, src) );
6865
6866 ins_pipe(ialu_imm);
6867 %}
6868
6869 // Load Pointer Constant
6870
6871 instruct loadConP(iRegPNoSp dst, immP con)
6872 %{
6873 match(Set dst con);
6874
6875 ins_cost(INSN_COST * 4);
6876 format %{
6877 "mov $dst, $con\t# ptr\n\t"
6878 %}
6879
6880 ins_encode(aarch64_enc_mov_p(dst, con));
6881
6882 ins_pipe(ialu_imm);
6883 %}
6884
6885 // Load Null Pointer Constant
6886
6887 instruct loadConP0(iRegPNoSp dst, immP0 con)
6888 %{
6889 match(Set dst con);
6890
6891 ins_cost(INSN_COST);
6892 format %{ "mov $dst, $con\t# nullptr ptr" %}
6893
6894 ins_encode(aarch64_enc_mov_p0(dst, con));
6895
6896 ins_pipe(ialu_imm);
6897 %}
6898
6899 // Load Pointer Constant One
6900
6901 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6902 %{
6903 match(Set dst con);
6904
6905 ins_cost(INSN_COST);
6906 format %{ "mov $dst, $con\t# nullptr ptr" %}
6907
6908 ins_encode(aarch64_enc_mov_p1(dst, con));
6909
6910 ins_pipe(ialu_imm);
6911 %}
6912
6913 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6914 %{
6915 match(Set dst con);
6916
6917 ins_cost(INSN_COST);
6918 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6919
6920 ins_encode %{
6921 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6922 %}
6923
6924 ins_pipe(ialu_imm);
6925 %}
6926
6927 // Load Narrow Pointer Constant
6928
6929 instruct loadConN(iRegNNoSp dst, immN con)
6930 %{
6931 match(Set dst con);
6932
6933 ins_cost(INSN_COST * 4);
6934 format %{ "mov $dst, $con\t# compressed ptr" %}
6935
6936 ins_encode(aarch64_enc_mov_n(dst, con));
6937
6938 ins_pipe(ialu_imm);
6939 %}
6940
6941 // Load Narrow Null Pointer Constant
6942
6943 instruct loadConN0(iRegNNoSp dst, immN0 con)
6944 %{
6945 match(Set dst con);
6946
6947 ins_cost(INSN_COST);
6948 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6949
6950 ins_encode(aarch64_enc_mov_n0(dst, con));
6951
6952 ins_pipe(ialu_imm);
6953 %}
6954
6955 // Load Narrow Klass Constant
6956
6957 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6958 %{
6959 match(Set dst con);
6960
6961 ins_cost(INSN_COST);
6962 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6963
6964 ins_encode(aarch64_enc_mov_nk(dst, con));
6965
6966 ins_pipe(ialu_imm);
6967 %}
6968
6969 // Load Packed Float Constant
6970
6971 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6972 match(Set dst con);
6973 ins_cost(INSN_COST * 4);
6974 format %{ "fmovs $dst, $con"%}
6975 ins_encode %{
6976 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6977 %}
6978
6979 ins_pipe(fp_imm_s);
6980 %}
6981
6982 // Load Float Constant
6983
6984 instruct loadConF(vRegF dst, immF con) %{
6985 match(Set dst con);
6986
6987 ins_cost(INSN_COST * 4);
6988
6989 format %{
6990 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6991 %}
6992
6993 ins_encode %{
6994 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6995 %}
6996
6997 ins_pipe(fp_load_constant_s);
6998 %}
6999
7000 // Load Packed Double Constant
7001
7002 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7003 match(Set dst con);
7004 ins_cost(INSN_COST);
7005 format %{ "fmovd $dst, $con"%}
7006 ins_encode %{
7007 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7008 %}
7009
7010 ins_pipe(fp_imm_d);
7011 %}
7012
7013 // Load Double Constant
7014
7015 instruct loadConD(vRegD dst, immD con) %{
7016 match(Set dst con);
7017
7018 ins_cost(INSN_COST * 5);
7019 format %{
7020 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7021 %}
7022
7023 ins_encode %{
7024 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7025 %}
7026
7027 ins_pipe(fp_load_constant_d);
7028 %}
7029
7030 // Load Half Float Constant
7031 instruct loadConH(vRegF dst, immH con) %{
7032 match(Set dst con);
7033 format %{ "mov rscratch1, $con\n\t"
7034 "fmov $dst, rscratch1"
7035 %}
7036 ins_encode %{
7037 __ movw(rscratch1, (uint32_t)$con$$constant);
7038 __ fmovs($dst$$FloatRegister, rscratch1);
7039 %}
7040 ins_pipe(pipe_class_default);
7041 %}
7042
7043 // Store Instructions
7044
7045 // Store Byte
7046 instruct storeB(iRegIorL2I src, memory1 mem)
7047 %{
7048 match(Set mem (StoreB mem src));
7049 predicate(!needs_releasing_store(n));
7050
7051 ins_cost(INSN_COST);
7052 format %{ "strb $src, $mem\t# byte" %}
7053
7054 ins_encode(aarch64_enc_strb(src, mem));
7055
7056 ins_pipe(istore_reg_mem);
7057 %}
7058
7059
7060 instruct storeimmB0(immI0 zero, memory1 mem)
7061 %{
7062 match(Set mem (StoreB mem zero));
7063 predicate(!needs_releasing_store(n));
7064
7065 ins_cost(INSN_COST);
7066 format %{ "strb rscractch2, $mem\t# byte" %}
7067
7068 ins_encode(aarch64_enc_strb0(mem));
7069
7070 ins_pipe(istore_mem);
7071 %}
7072
7073 // Store Char/Short
7074 instruct storeC(iRegIorL2I src, memory2 mem)
7075 %{
7076 match(Set mem (StoreC mem src));
7077 predicate(!needs_releasing_store(n));
7078
7079 ins_cost(INSN_COST);
7080 format %{ "strh $src, $mem\t# short" %}
7081
7082 ins_encode(aarch64_enc_strh(src, mem));
7083
7084 ins_pipe(istore_reg_mem);
7085 %}
7086
7087 instruct storeimmC0(immI0 zero, memory2 mem)
7088 %{
7089 match(Set mem (StoreC mem zero));
7090 predicate(!needs_releasing_store(n));
7091
7092 ins_cost(INSN_COST);
7093 format %{ "strh zr, $mem\t# short" %}
7094
7095 ins_encode(aarch64_enc_strh0(mem));
7096
7097 ins_pipe(istore_mem);
7098 %}
7099
7100 // Store Integer
7101
7102 instruct storeI(iRegIorL2I src, memory4 mem)
7103 %{
7104 match(Set mem(StoreI mem src));
7105 predicate(!needs_releasing_store(n));
7106
7107 ins_cost(INSN_COST);
7108 format %{ "strw $src, $mem\t# int" %}
7109
7110 ins_encode(aarch64_enc_strw(src, mem));
7111
7112 ins_pipe(istore_reg_mem);
7113 %}
7114
7115 instruct storeimmI0(immI0 zero, memory4 mem)
7116 %{
7117 match(Set mem(StoreI mem zero));
7118 predicate(!needs_releasing_store(n));
7119
7120 ins_cost(INSN_COST);
7121 format %{ "strw zr, $mem\t# int" %}
7122
7123 ins_encode(aarch64_enc_strw0(mem));
7124
7125 ins_pipe(istore_mem);
7126 %}
7127
7128 // Store Long (64 bit signed)
7129 instruct storeL(iRegL src, memory8 mem)
7130 %{
7131 match(Set mem (StoreL mem src));
7132 predicate(!needs_releasing_store(n));
7133
7134 ins_cost(INSN_COST);
7135 format %{ "str $src, $mem\t# int" %}
7136
7137 ins_encode(aarch64_enc_str(src, mem));
7138
7139 ins_pipe(istore_reg_mem);
7140 %}
7141
7142 // Store Long (64 bit signed)
7143 instruct storeimmL0(immL0 zero, memory8 mem)
7144 %{
7145 match(Set mem (StoreL mem zero));
7146 predicate(!needs_releasing_store(n));
7147
7148 ins_cost(INSN_COST);
7149 format %{ "str zr, $mem\t# int" %}
7150
7151 ins_encode(aarch64_enc_str0(mem));
7152
7153 ins_pipe(istore_mem);
7154 %}
7155
7156 // Store Pointer
7157 instruct storeP(iRegP src, memory8 mem)
7158 %{
7159 match(Set mem (StoreP mem src));
7160 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7161
7162 ins_cost(INSN_COST);
7163 format %{ "str $src, $mem\t# ptr" %}
7164
7165 ins_encode(aarch64_enc_str(src, mem));
7166
7167 ins_pipe(istore_reg_mem);
7168 %}
7169
7170 // Store Pointer
7171 instruct storeimmP0(immP0 zero, memory8 mem)
7172 %{
7173 match(Set mem (StoreP mem zero));
7174 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7175
7176 ins_cost(INSN_COST);
7177 format %{ "str zr, $mem\t# ptr" %}
7178
7179 ins_encode(aarch64_enc_str0(mem));
7180
7181 ins_pipe(istore_mem);
7182 %}
7183
7184 // Store Compressed Pointer
7185 instruct storeN(iRegN src, memory4 mem)
7186 %{
7187 match(Set mem (StoreN mem src));
7188 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7189
7190 ins_cost(INSN_COST);
7191 format %{ "strw $src, $mem\t# compressed ptr" %}
7192
7193 ins_encode(aarch64_enc_strw(src, mem));
7194
7195 ins_pipe(istore_reg_mem);
7196 %}
7197
7198 instruct storeImmN0(immN0 zero, memory4 mem)
7199 %{
7200 match(Set mem (StoreN mem zero));
7201 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7202
7203 ins_cost(INSN_COST);
7204 format %{ "strw zr, $mem\t# compressed ptr" %}
7205
7206 ins_encode(aarch64_enc_strw0(mem));
7207
7208 ins_pipe(istore_mem);
7209 %}
7210
7211 // Store Float
7212 instruct storeF(vRegF src, memory4 mem)
7213 %{
7214 match(Set mem (StoreF mem src));
7215 predicate(!needs_releasing_store(n));
7216
7217 ins_cost(INSN_COST);
7218 format %{ "strs $src, $mem\t# float" %}
7219
7220 ins_encode( aarch64_enc_strs(src, mem) );
7221
7222 ins_pipe(pipe_class_memory);
7223 %}
7224
7225 // TODO
7226 // implement storeImmF0 and storeFImmPacked
7227
7228 // Store Double
7229 instruct storeD(vRegD src, memory8 mem)
7230 %{
7231 match(Set mem (StoreD mem src));
7232 predicate(!needs_releasing_store(n));
7233
7234 ins_cost(INSN_COST);
7235 format %{ "strd $src, $mem\t# double" %}
7236
7237 ins_encode( aarch64_enc_strd(src, mem) );
7238
7239 ins_pipe(pipe_class_memory);
7240 %}
7241
7242 // Store Compressed Klass Pointer
7243 instruct storeNKlass(iRegN src, memory4 mem)
7244 %{
7245 predicate(!needs_releasing_store(n));
7246 match(Set mem (StoreNKlass mem src));
7247
7248 ins_cost(INSN_COST);
7249 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7250
7251 ins_encode(aarch64_enc_strw(src, mem));
7252
7253 ins_pipe(istore_reg_mem);
7254 %}
7255
7256 // TODO
7257 // implement storeImmD0 and storeDImmPacked
7258
7259 // prefetch instructions
7260 // Must be safe to execute with invalid address (cannot fault).
7261
7262 instruct prefetchalloc( memory8 mem ) %{
7263 match(PrefetchAllocation mem);
7264
7265 ins_cost(INSN_COST);
7266 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7267
7268 ins_encode( aarch64_enc_prefetchw(mem) );
7269
7270 ins_pipe(iload_prefetch);
7271 %}
7272
7273 // ---------------- volatile loads and stores ----------------
7274
7275 // Load Byte (8 bit signed)
7276 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7277 %{
7278 match(Set dst (LoadB mem));
7279
7280 ins_cost(VOLATILE_REF_COST);
7281 format %{ "ldarsb $dst, $mem\t# byte" %}
7282
7283 ins_encode(aarch64_enc_ldarsb(dst, mem));
7284
7285 ins_pipe(pipe_serial);
7286 %}
7287
7288 // Load Byte (8 bit signed) into long
7289 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7290 %{
7291 match(Set dst (ConvI2L (LoadB mem)));
7292
7293 ins_cost(VOLATILE_REF_COST);
7294 format %{ "ldarsb $dst, $mem\t# byte" %}
7295
7296 ins_encode(aarch64_enc_ldarsb(dst, mem));
7297
7298 ins_pipe(pipe_serial);
7299 %}
7300
7301 // Load Byte (8 bit unsigned)
7302 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7303 %{
7304 match(Set dst (LoadUB mem));
7305
7306 ins_cost(VOLATILE_REF_COST);
7307 format %{ "ldarb $dst, $mem\t# byte" %}
7308
7309 ins_encode(aarch64_enc_ldarb(dst, mem));
7310
7311 ins_pipe(pipe_serial);
7312 %}
7313
7314 // Load Byte (8 bit unsigned) into long
7315 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7316 %{
7317 match(Set dst (ConvI2L (LoadUB mem)));
7318
7319 ins_cost(VOLATILE_REF_COST);
7320 format %{ "ldarb $dst, $mem\t# byte" %}
7321
7322 ins_encode(aarch64_enc_ldarb(dst, mem));
7323
7324 ins_pipe(pipe_serial);
7325 %}
7326
7327 // Load Short (16 bit signed)
7328 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7329 %{
7330 match(Set dst (LoadS mem));
7331
7332 ins_cost(VOLATILE_REF_COST);
7333 format %{ "ldarshw $dst, $mem\t# short" %}
7334
7335 ins_encode(aarch64_enc_ldarshw(dst, mem));
7336
7337 ins_pipe(pipe_serial);
7338 %}
7339
7340 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7341 %{
7342 match(Set dst (LoadUS mem));
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "ldarhw $dst, $mem\t# short" %}
7346
7347 ins_encode(aarch64_enc_ldarhw(dst, mem));
7348
7349 ins_pipe(pipe_serial);
7350 %}
7351
7352 // Load Short/Char (16 bit unsigned) into long
7353 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (ConvI2L (LoadUS mem)));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldarh $dst, $mem\t# short" %}
7359
7360 ins_encode(aarch64_enc_ldarh(dst, mem));
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Short/Char (16 bit signed) into long
7366 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set dst (ConvI2L (LoadS mem)));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldarh $dst, $mem\t# short" %}
7372
7373 ins_encode(aarch64_enc_ldarsh(dst, mem));
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 // Load Integer (32 bit signed)
7379 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7380 %{
7381 match(Set dst (LoadI mem));
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "ldarw $dst, $mem\t# int" %}
7385
7386 ins_encode(aarch64_enc_ldarw(dst, mem));
7387
7388 ins_pipe(pipe_serial);
7389 %}
7390
7391 // Load Integer (32 bit unsigned) into long
7392 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7393 %{
7394 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7395
7396 ins_cost(VOLATILE_REF_COST);
7397 format %{ "ldarw $dst, $mem\t# int" %}
7398
7399 ins_encode(aarch64_enc_ldarw(dst, mem));
7400
7401 ins_pipe(pipe_serial);
7402 %}
7403
7404 // Load Long (64 bit signed)
7405 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7406 %{
7407 match(Set dst (LoadL mem));
7408
7409 ins_cost(VOLATILE_REF_COST);
7410 format %{ "ldar $dst, $mem\t# int" %}
7411
7412 ins_encode(aarch64_enc_ldar(dst, mem));
7413
7414 ins_pipe(pipe_serial);
7415 %}
7416
7417 // Load Pointer
7418 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7419 %{
7420 match(Set dst (LoadP mem));
7421 predicate(n->as_Load()->barrier_data() == 0);
7422
7423 ins_cost(VOLATILE_REF_COST);
7424 format %{ "ldar $dst, $mem\t# ptr" %}
7425
7426 ins_encode(aarch64_enc_ldar(dst, mem));
7427
7428 ins_pipe(pipe_serial);
7429 %}
7430
7431 // Load Compressed Pointer
7432 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7433 %{
7434 match(Set dst (LoadN mem));
7435 predicate(n->as_Load()->barrier_data() == 0);
7436
7437 ins_cost(VOLATILE_REF_COST);
7438 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7439
7440 ins_encode(aarch64_enc_ldarw(dst, mem));
7441
7442 ins_pipe(pipe_serial);
7443 %}
7444
7445 // Load Float
7446 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7447 %{
7448 match(Set dst (LoadF mem));
7449
7450 ins_cost(VOLATILE_REF_COST);
7451 format %{ "ldars $dst, $mem\t# float" %}
7452
7453 ins_encode( aarch64_enc_fldars(dst, mem) );
7454
7455 ins_pipe(pipe_serial);
7456 %}
7457
7458 // Load Double
7459 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7460 %{
7461 match(Set dst (LoadD mem));
7462
7463 ins_cost(VOLATILE_REF_COST);
7464 format %{ "ldard $dst, $mem\t# double" %}
7465
7466 ins_encode( aarch64_enc_fldard(dst, mem) );
7467
7468 ins_pipe(pipe_serial);
7469 %}
7470
7471 // Store Byte
7472 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7473 %{
7474 match(Set mem (StoreB mem src));
7475
7476 ins_cost(VOLATILE_REF_COST);
7477 format %{ "stlrb $src, $mem\t# byte" %}
7478
7479 ins_encode(aarch64_enc_stlrb(src, mem));
7480
7481 ins_pipe(pipe_class_memory);
7482 %}
7483
7484 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7485 %{
7486 match(Set mem (StoreB mem zero));
7487
7488 ins_cost(VOLATILE_REF_COST);
7489 format %{ "stlrb zr, $mem\t# byte" %}
7490
7491 ins_encode(aarch64_enc_stlrb0(mem));
7492
7493 ins_pipe(pipe_class_memory);
7494 %}
7495
7496 // Store Char/Short
7497 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7498 %{
7499 match(Set mem (StoreC mem src));
7500
7501 ins_cost(VOLATILE_REF_COST);
7502 format %{ "stlrh $src, $mem\t# short" %}
7503
7504 ins_encode(aarch64_enc_stlrh(src, mem));
7505
7506 ins_pipe(pipe_class_memory);
7507 %}
7508
7509 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7510 %{
7511 match(Set mem (StoreC mem zero));
7512
7513 ins_cost(VOLATILE_REF_COST);
7514 format %{ "stlrh zr, $mem\t# short" %}
7515
7516 ins_encode(aarch64_enc_stlrh0(mem));
7517
7518 ins_pipe(pipe_class_memory);
7519 %}
7520
7521 // Store Integer
7522
7523 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7524 %{
7525 match(Set mem(StoreI mem src));
7526
7527 ins_cost(VOLATILE_REF_COST);
7528 format %{ "stlrw $src, $mem\t# int" %}
7529
7530 ins_encode(aarch64_enc_stlrw(src, mem));
7531
7532 ins_pipe(pipe_class_memory);
7533 %}
7534
7535 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7536 %{
7537 match(Set mem(StoreI mem zero));
7538
7539 ins_cost(VOLATILE_REF_COST);
7540 format %{ "stlrw zr, $mem\t# int" %}
7541
7542 ins_encode(aarch64_enc_stlrw0(mem));
7543
7544 ins_pipe(pipe_class_memory);
7545 %}
7546
7547 // Store Long (64 bit signed)
7548 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7549 %{
7550 match(Set mem (StoreL mem src));
7551
7552 ins_cost(VOLATILE_REF_COST);
7553 format %{ "stlr $src, $mem\t# int" %}
7554
7555 ins_encode(aarch64_enc_stlr(src, mem));
7556
7557 ins_pipe(pipe_class_memory);
7558 %}
7559
7560 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7561 %{
7562 match(Set mem (StoreL mem zero));
7563
7564 ins_cost(VOLATILE_REF_COST);
7565 format %{ "stlr zr, $mem\t# int" %}
7566
7567 ins_encode(aarch64_enc_stlr0(mem));
7568
7569 ins_pipe(pipe_class_memory);
7570 %}
7571
7572 // Store Pointer
7573 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7574 %{
7575 match(Set mem (StoreP mem src));
7576 predicate(n->as_Store()->barrier_data() == 0);
7577
7578 ins_cost(VOLATILE_REF_COST);
7579 format %{ "stlr $src, $mem\t# ptr" %}
7580
7581 ins_encode(aarch64_enc_stlr(src, mem));
7582
7583 ins_pipe(pipe_class_memory);
7584 %}
7585
7586 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7587 %{
7588 match(Set mem (StoreP mem zero));
7589 predicate(n->as_Store()->barrier_data() == 0);
7590
7591 ins_cost(VOLATILE_REF_COST);
7592 format %{ "stlr zr, $mem\t# ptr" %}
7593
7594 ins_encode(aarch64_enc_stlr0(mem));
7595
7596 ins_pipe(pipe_class_memory);
7597 %}
7598
7599 // Store Compressed Pointer
7600 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7601 %{
7602 match(Set mem (StoreN mem src));
7603 predicate(n->as_Store()->barrier_data() == 0);
7604
7605 ins_cost(VOLATILE_REF_COST);
7606 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7607
7608 ins_encode(aarch64_enc_stlrw(src, mem));
7609
7610 ins_pipe(pipe_class_memory);
7611 %}
7612
7613 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7614 %{
7615 match(Set mem (StoreN mem zero));
7616 predicate(n->as_Store()->barrier_data() == 0);
7617
7618 ins_cost(VOLATILE_REF_COST);
7619 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7620
7621 ins_encode(aarch64_enc_stlrw0(mem));
7622
7623 ins_pipe(pipe_class_memory);
7624 %}
7625
7626 // Store Float
7627 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7628 %{
7629 match(Set mem (StoreF mem src));
7630
7631 ins_cost(VOLATILE_REF_COST);
7632 format %{ "stlrs $src, $mem\t# float" %}
7633
7634 ins_encode( aarch64_enc_fstlrs(src, mem) );
7635
7636 ins_pipe(pipe_class_memory);
7637 %}
7638
7639 // TODO
7640 // implement storeImmF0 and storeFImmPacked
7641
7642 // Store Double
7643 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7644 %{
7645 match(Set mem (StoreD mem src));
7646
7647 ins_cost(VOLATILE_REF_COST);
7648 format %{ "stlrd $src, $mem\t# double" %}
7649
7650 ins_encode( aarch64_enc_fstlrd(src, mem) );
7651
7652 ins_pipe(pipe_class_memory);
7653 %}
7654
7655 // ---------------- end of volatile loads and stores ----------------
7656
7657 instruct cacheWB(indirect addr)
7658 %{
7659 predicate(VM_Version::supports_data_cache_line_flush());
7660 match(CacheWB addr);
7661
7662 ins_cost(100);
7663 format %{"cache wb $addr" %}
7664 ins_encode %{
7665 assert($addr->index_position() < 0, "should be");
7666 assert($addr$$disp == 0, "should be");
7667 __ cache_wb(Address($addr$$base$$Register, 0));
7668 %}
7669 ins_pipe(pipe_slow); // XXX
7670 %}
7671
7672 instruct cacheWBPreSync()
7673 %{
7674 predicate(VM_Version::supports_data_cache_line_flush());
7675 match(CacheWBPreSync);
7676
7677 ins_cost(100);
7678 format %{"cache wb presync" %}
7679 ins_encode %{
7680 __ cache_wbsync(true);
7681 %}
7682 ins_pipe(pipe_slow); // XXX
7683 %}
7684
7685 instruct cacheWBPostSync()
7686 %{
7687 predicate(VM_Version::supports_data_cache_line_flush());
7688 match(CacheWBPostSync);
7689
7690 ins_cost(100);
7691 format %{"cache wb postsync" %}
7692 ins_encode %{
7693 __ cache_wbsync(false);
7694 %}
7695 ins_pipe(pipe_slow); // XXX
7696 %}
7697
7698 // ============================================================================
7699 // BSWAP Instructions
7700
7701 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7702 match(Set dst (ReverseBytesI src));
7703
7704 ins_cost(INSN_COST);
7705 format %{ "revw $dst, $src" %}
7706
7707 ins_encode %{
7708 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7709 %}
7710
7711 ins_pipe(ialu_reg);
7712 %}
7713
7714 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7715 match(Set dst (ReverseBytesL src));
7716
7717 ins_cost(INSN_COST);
7718 format %{ "rev $dst, $src" %}
7719
7720 ins_encode %{
7721 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7722 %}
7723
7724 ins_pipe(ialu_reg);
7725 %}
7726
7727 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7728 match(Set dst (ReverseBytesUS src));
7729
7730 ins_cost(INSN_COST);
7731 format %{ "rev16w $dst, $src" %}
7732
7733 ins_encode %{
7734 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7735 %}
7736
7737 ins_pipe(ialu_reg);
7738 %}
7739
7740 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7741 match(Set dst (ReverseBytesS src));
7742
7743 ins_cost(INSN_COST);
7744 format %{ "rev16w $dst, $src\n\t"
7745 "sbfmw $dst, $dst, #0, #15" %}
7746
7747 ins_encode %{
7748 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7749 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7750 %}
7751
7752 ins_pipe(ialu_reg);
7753 %}
7754
7755 // ============================================================================
7756 // Zero Count Instructions
7757
7758 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7759 match(Set dst (CountLeadingZerosI src));
7760
7761 ins_cost(INSN_COST);
7762 format %{ "clzw $dst, $src" %}
7763 ins_encode %{
7764 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7765 %}
7766
7767 ins_pipe(ialu_reg);
7768 %}
7769
7770 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7771 match(Set dst (CountLeadingZerosL src));
7772
7773 ins_cost(INSN_COST);
7774 format %{ "clz $dst, $src" %}
7775 ins_encode %{
7776 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7777 %}
7778
7779 ins_pipe(ialu_reg);
7780 %}
7781
7782 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7783 match(Set dst (CountTrailingZerosI src));
7784
7785 ins_cost(INSN_COST * 2);
7786 format %{ "rbitw $dst, $src\n\t"
7787 "clzw $dst, $dst" %}
7788 ins_encode %{
7789 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7790 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7791 %}
7792
7793 ins_pipe(ialu_reg);
7794 %}
7795
7796 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7797 match(Set dst (CountTrailingZerosL src));
7798
7799 ins_cost(INSN_COST * 2);
7800 format %{ "rbit $dst, $src\n\t"
7801 "clz $dst, $dst" %}
7802 ins_encode %{
7803 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7804 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7805 %}
7806
7807 ins_pipe(ialu_reg);
7808 %}
7809
7810 //---------- Population Count Instructions -------------------------------------
7811 //
7812
7813 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7814 match(Set dst (PopCountI src));
7815 effect(TEMP tmp);
7816 ins_cost(INSN_COST * 13);
7817
7818 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7819 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7820 "addv $tmp, $tmp\t# vector (8B)\n\t"
7821 "mov $dst, $tmp\t# vector (1D)" %}
7822 ins_encode %{
7823 __ fmovs($tmp$$FloatRegister, $src$$Register);
7824 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7825 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7826 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7827 %}
7828
7829 ins_pipe(pipe_class_default);
7830 %}
7831
7832 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7833 match(Set dst (PopCountI (LoadI mem)));
7834 effect(TEMP tmp);
7835 ins_cost(INSN_COST * 13);
7836
7837 format %{ "ldrs $tmp, $mem\n\t"
7838 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7839 "addv $tmp, $tmp\t# vector (8B)\n\t"
7840 "mov $dst, $tmp\t# vector (1D)" %}
7841 ins_encode %{
7842 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7843 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7844 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7845 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7846 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7847 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7848 %}
7849
7850 ins_pipe(pipe_class_default);
7851 %}
7852
7853 // Note: Long.bitCount(long) returns an int.
7854 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7855 match(Set dst (PopCountL src));
7856 effect(TEMP tmp);
7857 ins_cost(INSN_COST * 13);
7858
7859 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7860 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7861 "addv $tmp, $tmp\t# vector (8B)\n\t"
7862 "mov $dst, $tmp\t# vector (1D)" %}
7863 ins_encode %{
7864 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7865 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7866 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7867 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7868 %}
7869
7870 ins_pipe(pipe_class_default);
7871 %}
7872
7873 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7874 match(Set dst (PopCountL (LoadL mem)));
7875 effect(TEMP tmp);
7876 ins_cost(INSN_COST * 13);
7877
7878 format %{ "ldrd $tmp, $mem\n\t"
7879 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7880 "addv $tmp, $tmp\t# vector (8B)\n\t"
7881 "mov $dst, $tmp\t# vector (1D)" %}
7882 ins_encode %{
7883 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7884 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7885 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7886 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7887 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7888 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7889 %}
7890
7891 ins_pipe(pipe_class_default);
7892 %}
7893
7894 // ============================================================================
7895 // VerifyVectorAlignment Instruction
7896
7897 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7898 match(Set addr (VerifyVectorAlignment addr mask));
7899 effect(KILL cr);
7900 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7901 ins_encode %{
7902 Label Lskip;
7903 // check if masked bits of addr are zero
7904 __ tst($addr$$Register, $mask$$constant);
7905 __ br(Assembler::EQ, Lskip);
7906 __ stop("verify_vector_alignment found a misaligned vector memory access");
7907 __ bind(Lskip);
7908 %}
7909 ins_pipe(pipe_slow);
7910 %}
7911
7912 // ============================================================================
7913 // MemBar Instruction
7914
7915 instruct load_fence() %{
7916 match(LoadFence);
7917 ins_cost(VOLATILE_REF_COST);
7918
7919 format %{ "load_fence" %}
7920
7921 ins_encode %{
7922 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7923 %}
7924 ins_pipe(pipe_serial);
7925 %}
7926
7927 instruct unnecessary_membar_acquire() %{
7928 predicate(unnecessary_acquire(n));
7929 match(MemBarAcquire);
7930 ins_cost(0);
7931
7932 format %{ "membar_acquire (elided)" %}
7933
7934 ins_encode %{
7935 __ block_comment("membar_acquire (elided)");
7936 %}
7937
7938 ins_pipe(pipe_class_empty);
7939 %}
7940
7941 instruct membar_acquire() %{
7942 match(MemBarAcquire);
7943 ins_cost(VOLATILE_REF_COST);
7944
7945 format %{ "membar_acquire\n\t"
7946 "dmb ishld" %}
7947
7948 ins_encode %{
7949 __ block_comment("membar_acquire");
7950 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7951 %}
7952
7953 ins_pipe(pipe_serial);
7954 %}
7955
7956
7957 instruct membar_acquire_lock() %{
7958 match(MemBarAcquireLock);
7959 ins_cost(VOLATILE_REF_COST);
7960
7961 format %{ "membar_acquire_lock (elided)" %}
7962
7963 ins_encode %{
7964 __ block_comment("membar_acquire_lock (elided)");
7965 %}
7966
7967 ins_pipe(pipe_serial);
7968 %}
7969
7970 instruct store_fence() %{
7971 match(StoreFence);
7972 ins_cost(VOLATILE_REF_COST);
7973
7974 format %{ "store_fence" %}
7975
7976 ins_encode %{
7977 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7978 %}
7979 ins_pipe(pipe_serial);
7980 %}
7981
7982 instruct unnecessary_membar_release() %{
7983 predicate(unnecessary_release(n));
7984 match(MemBarRelease);
7985 ins_cost(0);
7986
7987 format %{ "membar_release (elided)" %}
7988
7989 ins_encode %{
7990 __ block_comment("membar_release (elided)");
7991 %}
7992 ins_pipe(pipe_serial);
7993 %}
7994
7995 instruct membar_release() %{
7996 match(MemBarRelease);
7997 ins_cost(VOLATILE_REF_COST);
7998
7999 format %{ "membar_release\n\t"
8000 "dmb ishst\n\tdmb ishld" %}
8001
8002 ins_encode %{
8003 __ block_comment("membar_release");
8004 // These will be merged if AlwaysMergeDMB is enabled.
8005 __ membar(Assembler::StoreStore);
8006 __ membar(Assembler::LoadStore);
8007 %}
8008 ins_pipe(pipe_serial);
8009 %}
8010
8011 instruct membar_storestore() %{
8012 match(MemBarStoreStore);
8013 match(StoreStoreFence);
8014 ins_cost(VOLATILE_REF_COST);
8015
8016 format %{ "MEMBAR-store-store" %}
8017
8018 ins_encode %{
8019 __ membar(Assembler::StoreStore);
8020 %}
8021 ins_pipe(pipe_serial);
8022 %}
8023
8024 instruct membar_release_lock() %{
8025 match(MemBarReleaseLock);
8026 ins_cost(VOLATILE_REF_COST);
8027
8028 format %{ "membar_release_lock (elided)" %}
8029
8030 ins_encode %{
8031 __ block_comment("membar_release_lock (elided)");
8032 %}
8033
8034 ins_pipe(pipe_serial);
8035 %}
8036
8037 instruct unnecessary_membar_volatile() %{
8038 predicate(unnecessary_volatile(n));
8039 match(MemBarVolatile);
8040 ins_cost(0);
8041
8042 format %{ "membar_volatile (elided)" %}
8043
8044 ins_encode %{
8045 __ block_comment("membar_volatile (elided)");
8046 %}
8047
8048 ins_pipe(pipe_serial);
8049 %}
8050
8051 instruct membar_volatile() %{
8052 match(MemBarVolatile);
8053 ins_cost(VOLATILE_REF_COST*100);
8054
8055 format %{ "membar_volatile\n\t"
8056 "dmb ish"%}
8057
8058 ins_encode %{
8059 __ block_comment("membar_volatile");
8060 __ membar(Assembler::StoreLoad);
8061 %}
8062
8063 ins_pipe(pipe_serial);
8064 %}
8065
8066 // ============================================================================
8067 // Cast/Convert Instructions
8068
8069 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8070 match(Set dst (CastX2P src));
8071
8072 ins_cost(INSN_COST);
8073 format %{ "mov $dst, $src\t# long -> ptr" %}
8074
8075 ins_encode %{
8076 if ($dst$$reg != $src$$reg) {
8077 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8078 }
8079 %}
8080
8081 ins_pipe(ialu_reg);
8082 %}
8083
8084 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8085 match(Set dst (CastP2X src));
8086
8087 ins_cost(INSN_COST);
8088 format %{ "mov $dst, $src\t# ptr -> long" %}
8089
8090 ins_encode %{
8091 if ($dst$$reg != $src$$reg) {
8092 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8093 }
8094 %}
8095
8096 ins_pipe(ialu_reg);
8097 %}
8098
8099 // Convert oop into int for vectors alignment masking
8100 instruct convP2I(iRegINoSp dst, iRegP src) %{
8101 match(Set dst (ConvL2I (CastP2X src)));
8102
8103 ins_cost(INSN_COST);
8104 format %{ "movw $dst, $src\t# ptr -> int" %}
8105 ins_encode %{
8106 __ movw($dst$$Register, $src$$Register);
8107 %}
8108
8109 ins_pipe(ialu_reg);
8110 %}
8111
8112 // Convert compressed oop into int for vectors alignment masking
8113 // in case of 32bit oops (heap < 4Gb).
8114 instruct convN2I(iRegINoSp dst, iRegN src)
8115 %{
8116 predicate(CompressedOops::shift() == 0);
8117 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8118
8119 ins_cost(INSN_COST);
8120 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8121 ins_encode %{
8122 __ movw($dst$$Register, $src$$Register);
8123 %}
8124
8125 ins_pipe(ialu_reg);
8126 %}
8127
8128
8129 // Convert oop pointer into compressed form
8130 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8131 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8132 match(Set dst (EncodeP src));
8133 effect(KILL cr);
8134 ins_cost(INSN_COST * 3);
8135 format %{ "encode_heap_oop $dst, $src" %}
8136 ins_encode %{
8137 Register s = $src$$Register;
8138 Register d = $dst$$Register;
8139 __ encode_heap_oop(d, s);
8140 %}
8141 ins_pipe(ialu_reg);
8142 %}
8143
8144 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8145 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8146 match(Set dst (EncodeP src));
8147 ins_cost(INSN_COST * 3);
8148 format %{ "encode_heap_oop_not_null $dst, $src" %}
8149 ins_encode %{
8150 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8151 %}
8152 ins_pipe(ialu_reg);
8153 %}
8154
8155 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8156 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8157 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8158 match(Set dst (DecodeN src));
8159 ins_cost(INSN_COST * 3);
8160 format %{ "decode_heap_oop $dst, $src" %}
8161 ins_encode %{
8162 Register s = $src$$Register;
8163 Register d = $dst$$Register;
8164 __ decode_heap_oop(d, s);
8165 %}
8166 ins_pipe(ialu_reg);
8167 %}
8168
8169 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8170 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8171 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8172 match(Set dst (DecodeN src));
8173 ins_cost(INSN_COST * 3);
8174 format %{ "decode_heap_oop_not_null $dst, $src" %}
8175 ins_encode %{
8176 Register s = $src$$Register;
8177 Register d = $dst$$Register;
8178 __ decode_heap_oop_not_null(d, s);
8179 %}
8180 ins_pipe(ialu_reg);
8181 %}
8182
8183 // n.b. AArch64 implementations of encode_klass_not_null and
8184 // decode_klass_not_null do not modify the flags register so, unlike
8185 // Intel, we don't kill CR as a side effect here
8186
8187 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8188 match(Set dst (EncodePKlass src));
8189
8190 ins_cost(INSN_COST * 3);
8191 format %{ "encode_klass_not_null $dst,$src" %}
8192
8193 ins_encode %{
8194 Register src_reg = as_Register($src$$reg);
8195 Register dst_reg = as_Register($dst$$reg);
8196 __ encode_klass_not_null(dst_reg, src_reg);
8197 %}
8198
8199 ins_pipe(ialu_reg);
8200 %}
8201
8202 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8203 match(Set dst (DecodeNKlass src));
8204
8205 ins_cost(INSN_COST * 3);
8206 format %{ "decode_klass_not_null $dst,$src" %}
8207
8208 ins_encode %{
8209 Register src_reg = as_Register($src$$reg);
8210 Register dst_reg = as_Register($dst$$reg);
8211 if (dst_reg != src_reg) {
8212 __ decode_klass_not_null(dst_reg, src_reg);
8213 } else {
8214 __ decode_klass_not_null(dst_reg);
8215 }
8216 %}
8217
8218 ins_pipe(ialu_reg);
8219 %}
8220
8221 instruct checkCastPP(iRegPNoSp dst)
8222 %{
8223 match(Set dst (CheckCastPP dst));
8224
8225 size(0);
8226 format %{ "# checkcastPP of $dst" %}
8227 ins_encode(/* empty encoding */);
8228 ins_pipe(pipe_class_empty);
8229 %}
8230
8231 instruct castPP(iRegPNoSp dst)
8232 %{
8233 match(Set dst (CastPP dst));
8234
8235 size(0);
8236 format %{ "# castPP of $dst" %}
8237 ins_encode(/* empty encoding */);
8238 ins_pipe(pipe_class_empty);
8239 %}
8240
8241 instruct castII(iRegI dst)
8242 %{
8243 predicate(VerifyConstraintCasts == 0);
8244 match(Set dst (CastII dst));
8245
8246 size(0);
8247 format %{ "# castII of $dst" %}
8248 ins_encode(/* empty encoding */);
8249 ins_cost(0);
8250 ins_pipe(pipe_class_empty);
8251 %}
8252
8253 instruct castII_checked(iRegI dst, rFlagsReg cr)
8254 %{
8255 predicate(VerifyConstraintCasts > 0);
8256 match(Set dst (CastII dst));
8257 effect(KILL cr);
8258
8259 format %{ "# castII_checked of $dst" %}
8260 ins_encode %{
8261 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8262 %}
8263 ins_pipe(pipe_slow);
8264 %}
8265
8266 instruct castLL(iRegL dst)
8267 %{
8268 predicate(VerifyConstraintCasts == 0);
8269 match(Set dst (CastLL dst));
8270
8271 size(0);
8272 format %{ "# castLL of $dst" %}
8273 ins_encode(/* empty encoding */);
8274 ins_cost(0);
8275 ins_pipe(pipe_class_empty);
8276 %}
8277
8278 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8279 %{
8280 predicate(VerifyConstraintCasts > 0);
8281 match(Set dst (CastLL dst));
8282 effect(KILL cr);
8283
8284 format %{ "# castLL_checked of $dst" %}
8285 ins_encode %{
8286 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8287 %}
8288 ins_pipe(pipe_slow);
8289 %}
8290
8291 instruct castHH(vRegF dst)
8292 %{
8293 match(Set dst (CastHH dst));
8294 size(0);
8295 format %{ "# castHH of $dst" %}
8296 ins_encode(/* empty encoding */);
8297 ins_cost(0);
8298 ins_pipe(pipe_class_empty);
8299 %}
8300
8301 instruct castFF(vRegF dst)
8302 %{
8303 match(Set dst (CastFF dst));
8304
8305 size(0);
8306 format %{ "# castFF of $dst" %}
8307 ins_encode(/* empty encoding */);
8308 ins_cost(0);
8309 ins_pipe(pipe_class_empty);
8310 %}
8311
8312 instruct castDD(vRegD dst)
8313 %{
8314 match(Set dst (CastDD dst));
8315
8316 size(0);
8317 format %{ "# castDD of $dst" %}
8318 ins_encode(/* empty encoding */);
8319 ins_cost(0);
8320 ins_pipe(pipe_class_empty);
8321 %}
8322
8323 instruct castVV(vReg dst)
8324 %{
8325 match(Set dst (CastVV dst));
8326
8327 size(0);
8328 format %{ "# castVV of $dst" %}
8329 ins_encode(/* empty encoding */);
8330 ins_cost(0);
8331 ins_pipe(pipe_class_empty);
8332 %}
8333
8334 instruct castVVMask(pRegGov dst)
8335 %{
8336 match(Set dst (CastVV dst));
8337
8338 size(0);
8339 format %{ "# castVV of $dst" %}
8340 ins_encode(/* empty encoding */);
8341 ins_cost(0);
8342 ins_pipe(pipe_class_empty);
8343 %}
8344
8345 // Manifest a CmpU result in an integer register.
8346 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8347 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8348 %{
8349 match(Set dst (CmpU3 src1 src2));
8350 effect(KILL flags);
8351
8352 ins_cost(INSN_COST * 3);
8353 format %{
8354 "cmpw $src1, $src2\n\t"
8355 "csetw $dst, ne\n\t"
8356 "cnegw $dst, lo\t# CmpU3(reg)"
8357 %}
8358 ins_encode %{
8359 __ cmpw($src1$$Register, $src2$$Register);
8360 __ csetw($dst$$Register, Assembler::NE);
8361 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8362 %}
8363
8364 ins_pipe(pipe_class_default);
8365 %}
8366
8367 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8368 %{
8369 match(Set dst (CmpU3 src1 src2));
8370 effect(KILL flags);
8371
8372 ins_cost(INSN_COST * 3);
8373 format %{
8374 "subsw zr, $src1, $src2\n\t"
8375 "csetw $dst, ne\n\t"
8376 "cnegw $dst, lo\t# CmpU3(imm)"
8377 %}
8378 ins_encode %{
8379 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8380 __ csetw($dst$$Register, Assembler::NE);
8381 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8382 %}
8383
8384 ins_pipe(pipe_class_default);
8385 %}
8386
8387 // Manifest a CmpUL result in an integer register.
8388 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8389 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8390 %{
8391 match(Set dst (CmpUL3 src1 src2));
8392 effect(KILL flags);
8393
8394 ins_cost(INSN_COST * 3);
8395 format %{
8396 "cmp $src1, $src2\n\t"
8397 "csetw $dst, ne\n\t"
8398 "cnegw $dst, lo\t# CmpUL3(reg)"
8399 %}
8400 ins_encode %{
8401 __ cmp($src1$$Register, $src2$$Register);
8402 __ csetw($dst$$Register, Assembler::NE);
8403 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8404 %}
8405
8406 ins_pipe(pipe_class_default);
8407 %}
8408
8409 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8410 %{
8411 match(Set dst (CmpUL3 src1 src2));
8412 effect(KILL flags);
8413
8414 ins_cost(INSN_COST * 3);
8415 format %{
8416 "subs zr, $src1, $src2\n\t"
8417 "csetw $dst, ne\n\t"
8418 "cnegw $dst, lo\t# CmpUL3(imm)"
8419 %}
8420 ins_encode %{
8421 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8422 __ csetw($dst$$Register, Assembler::NE);
8423 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8424 %}
8425
8426 ins_pipe(pipe_class_default);
8427 %}
8428
8429 // Manifest a CmpL result in an integer register.
8430 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8431 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8432 %{
8433 match(Set dst (CmpL3 src1 src2));
8434 effect(KILL flags);
8435
8436 ins_cost(INSN_COST * 3);
8437 format %{
8438 "cmp $src1, $src2\n\t"
8439 "csetw $dst, ne\n\t"
8440 "cnegw $dst, lt\t# CmpL3(reg)"
8441 %}
8442 ins_encode %{
8443 __ cmp($src1$$Register, $src2$$Register);
8444 __ csetw($dst$$Register, Assembler::NE);
8445 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8446 %}
8447
8448 ins_pipe(pipe_class_default);
8449 %}
8450
8451 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8452 %{
8453 match(Set dst (CmpL3 src1 src2));
8454 effect(KILL flags);
8455
8456 ins_cost(INSN_COST * 3);
8457 format %{
8458 "subs zr, $src1, $src2\n\t"
8459 "csetw $dst, ne\n\t"
8460 "cnegw $dst, lt\t# CmpL3(imm)"
8461 %}
8462 ins_encode %{
8463 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8464 __ csetw($dst$$Register, Assembler::NE);
8465 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8466 %}
8467
8468 ins_pipe(pipe_class_default);
8469 %}
8470
8471 // ============================================================================
8472 // Conditional Move Instructions
8473
8474 // n.b. we have identical rules for both a signed compare op (cmpOp)
8475 // and an unsigned compare op (cmpOpU). it would be nice if we could
8476 // define an op class which merged both inputs and use it to type the
8477 // argument to a single rule. unfortunatelyt his fails because the
8478 // opclass does not live up to the COND_INTER interface of its
8479 // component operands. When the generic code tries to negate the
8480 // operand it ends up running the generci Machoper::negate method
8481 // which throws a ShouldNotHappen. So, we have to provide two flavours
8482 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8483
8484 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8485 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8486
8487 ins_cost(INSN_COST * 2);
8488 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8489
8490 ins_encode %{
8491 __ cselw(as_Register($dst$$reg),
8492 as_Register($src2$$reg),
8493 as_Register($src1$$reg),
8494 (Assembler::Condition)$cmp$$cmpcode);
8495 %}
8496
8497 ins_pipe(icond_reg_reg);
8498 %}
8499
8500 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8501 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8502
8503 ins_cost(INSN_COST * 2);
8504 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8505
8506 ins_encode %{
8507 __ cselw(as_Register($dst$$reg),
8508 as_Register($src2$$reg),
8509 as_Register($src1$$reg),
8510 (Assembler::Condition)$cmp$$cmpcode);
8511 %}
8512
8513 ins_pipe(icond_reg_reg);
8514 %}
8515
8516 // special cases where one arg is zero
8517
8518 // n.b. this is selected in preference to the rule above because it
8519 // avoids loading constant 0 into a source register
8520
8521 // TODO
8522 // we ought only to be able to cull one of these variants as the ideal
8523 // transforms ought always to order the zero consistently (to left/right?)
8524
8525 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8526 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8527
8528 ins_cost(INSN_COST * 2);
8529 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8530
8531 ins_encode %{
8532 __ cselw(as_Register($dst$$reg),
8533 as_Register($src$$reg),
8534 zr,
8535 (Assembler::Condition)$cmp$$cmpcode);
8536 %}
8537
8538 ins_pipe(icond_reg);
8539 %}
8540
8541 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8542 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8543
8544 ins_cost(INSN_COST * 2);
8545 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8546
8547 ins_encode %{
8548 __ cselw(as_Register($dst$$reg),
8549 as_Register($src$$reg),
8550 zr,
8551 (Assembler::Condition)$cmp$$cmpcode);
8552 %}
8553
8554 ins_pipe(icond_reg);
8555 %}
8556
8557 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8558 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8559
8560 ins_cost(INSN_COST * 2);
8561 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8562
8563 ins_encode %{
8564 __ cselw(as_Register($dst$$reg),
8565 zr,
8566 as_Register($src$$reg),
8567 (Assembler::Condition)$cmp$$cmpcode);
8568 %}
8569
8570 ins_pipe(icond_reg);
8571 %}
8572
8573 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8574 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8575
8576 ins_cost(INSN_COST * 2);
8577 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8578
8579 ins_encode %{
8580 __ cselw(as_Register($dst$$reg),
8581 zr,
8582 as_Register($src$$reg),
8583 (Assembler::Condition)$cmp$$cmpcode);
8584 %}
8585
8586 ins_pipe(icond_reg);
8587 %}
8588
8589 // special case for creating a boolean 0 or 1
8590
8591 // n.b. this is selected in preference to the rule above because it
8592 // avoids loading constants 0 and 1 into a source register
8593
8594 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8595 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8596
8597 ins_cost(INSN_COST * 2);
8598 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8599
8600 ins_encode %{
8601 // equivalently
8602 // cset(as_Register($dst$$reg),
8603 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8604 __ csincw(as_Register($dst$$reg),
8605 zr,
8606 zr,
8607 (Assembler::Condition)$cmp$$cmpcode);
8608 %}
8609
8610 ins_pipe(icond_none);
8611 %}
8612
8613 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8614 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8615
8616 ins_cost(INSN_COST * 2);
8617 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8618
8619 ins_encode %{
8620 // equivalently
8621 // cset(as_Register($dst$$reg),
8622 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8623 __ csincw(as_Register($dst$$reg),
8624 zr,
8625 zr,
8626 (Assembler::Condition)$cmp$$cmpcode);
8627 %}
8628
8629 ins_pipe(icond_none);
8630 %}
8631
8632 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8633 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8634
8635 ins_cost(INSN_COST * 2);
8636 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8637
8638 ins_encode %{
8639 __ csel(as_Register($dst$$reg),
8640 as_Register($src2$$reg),
8641 as_Register($src1$$reg),
8642 (Assembler::Condition)$cmp$$cmpcode);
8643 %}
8644
8645 ins_pipe(icond_reg_reg);
8646 %}
8647
8648 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8649 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8650
8651 ins_cost(INSN_COST * 2);
8652 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8653
8654 ins_encode %{
8655 __ csel(as_Register($dst$$reg),
8656 as_Register($src2$$reg),
8657 as_Register($src1$$reg),
8658 (Assembler::Condition)$cmp$$cmpcode);
8659 %}
8660
8661 ins_pipe(icond_reg_reg);
8662 %}
8663
8664 // special cases where one arg is zero
8665
8666 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8667 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8668
8669 ins_cost(INSN_COST * 2);
8670 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8671
8672 ins_encode %{
8673 __ csel(as_Register($dst$$reg),
8674 zr,
8675 as_Register($src$$reg),
8676 (Assembler::Condition)$cmp$$cmpcode);
8677 %}
8678
8679 ins_pipe(icond_reg);
8680 %}
8681
8682 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8683 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8684
8685 ins_cost(INSN_COST * 2);
8686 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8687
8688 ins_encode %{
8689 __ csel(as_Register($dst$$reg),
8690 zr,
8691 as_Register($src$$reg),
8692 (Assembler::Condition)$cmp$$cmpcode);
8693 %}
8694
8695 ins_pipe(icond_reg);
8696 %}
8697
8698 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8699 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8700
8701 ins_cost(INSN_COST * 2);
8702 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8703
8704 ins_encode %{
8705 __ csel(as_Register($dst$$reg),
8706 as_Register($src$$reg),
8707 zr,
8708 (Assembler::Condition)$cmp$$cmpcode);
8709 %}
8710
8711 ins_pipe(icond_reg);
8712 %}
8713
8714 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8715 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8716
8717 ins_cost(INSN_COST * 2);
8718 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8719
8720 ins_encode %{
8721 __ csel(as_Register($dst$$reg),
8722 as_Register($src$$reg),
8723 zr,
8724 (Assembler::Condition)$cmp$$cmpcode);
8725 %}
8726
8727 ins_pipe(icond_reg);
8728 %}
8729
8730 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8731 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8732
8733 ins_cost(INSN_COST * 2);
8734 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8735
8736 ins_encode %{
8737 __ csel(as_Register($dst$$reg),
8738 as_Register($src2$$reg),
8739 as_Register($src1$$reg),
8740 (Assembler::Condition)$cmp$$cmpcode);
8741 %}
8742
8743 ins_pipe(icond_reg_reg);
8744 %}
8745
8746 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8747 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8748
8749 ins_cost(INSN_COST * 2);
8750 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8751
8752 ins_encode %{
8753 __ csel(as_Register($dst$$reg),
8754 as_Register($src2$$reg),
8755 as_Register($src1$$reg),
8756 (Assembler::Condition)$cmp$$cmpcode);
8757 %}
8758
8759 ins_pipe(icond_reg_reg);
8760 %}
8761
8762 // special cases where one arg is zero
8763
8764 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8765 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8766
8767 ins_cost(INSN_COST * 2);
8768 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8769
8770 ins_encode %{
8771 __ csel(as_Register($dst$$reg),
8772 zr,
8773 as_Register($src$$reg),
8774 (Assembler::Condition)$cmp$$cmpcode);
8775 %}
8776
8777 ins_pipe(icond_reg);
8778 %}
8779
8780 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8781 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8782
8783 ins_cost(INSN_COST * 2);
8784 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8785
8786 ins_encode %{
8787 __ csel(as_Register($dst$$reg),
8788 zr,
8789 as_Register($src$$reg),
8790 (Assembler::Condition)$cmp$$cmpcode);
8791 %}
8792
8793 ins_pipe(icond_reg);
8794 %}
8795
8796 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8797 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8798
8799 ins_cost(INSN_COST * 2);
8800 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8801
8802 ins_encode %{
8803 __ csel(as_Register($dst$$reg),
8804 as_Register($src$$reg),
8805 zr,
8806 (Assembler::Condition)$cmp$$cmpcode);
8807 %}
8808
8809 ins_pipe(icond_reg);
8810 %}
8811
8812 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8813 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8814
8815 ins_cost(INSN_COST * 2);
8816 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8817
8818 ins_encode %{
8819 __ csel(as_Register($dst$$reg),
8820 as_Register($src$$reg),
8821 zr,
8822 (Assembler::Condition)$cmp$$cmpcode);
8823 %}
8824
8825 ins_pipe(icond_reg);
8826 %}
8827
8828 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8829 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8830
8831 ins_cost(INSN_COST * 2);
8832 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8833
8834 ins_encode %{
8835 __ cselw(as_Register($dst$$reg),
8836 as_Register($src2$$reg),
8837 as_Register($src1$$reg),
8838 (Assembler::Condition)$cmp$$cmpcode);
8839 %}
8840
8841 ins_pipe(icond_reg_reg);
8842 %}
8843
8844 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8845 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8846
8847 ins_cost(INSN_COST * 2);
8848 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8849
8850 ins_encode %{
8851 __ cselw(as_Register($dst$$reg),
8852 as_Register($src2$$reg),
8853 as_Register($src1$$reg),
8854 (Assembler::Condition)$cmp$$cmpcode);
8855 %}
8856
8857 ins_pipe(icond_reg_reg);
8858 %}
8859
8860 // special cases where one arg is zero
8861
8862 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8863 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8864
8865 ins_cost(INSN_COST * 2);
8866 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8867
8868 ins_encode %{
8869 __ cselw(as_Register($dst$$reg),
8870 zr,
8871 as_Register($src$$reg),
8872 (Assembler::Condition)$cmp$$cmpcode);
8873 %}
8874
8875 ins_pipe(icond_reg);
8876 %}
8877
8878 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8879 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8880
8881 ins_cost(INSN_COST * 2);
8882 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8883
8884 ins_encode %{
8885 __ cselw(as_Register($dst$$reg),
8886 zr,
8887 as_Register($src$$reg),
8888 (Assembler::Condition)$cmp$$cmpcode);
8889 %}
8890
8891 ins_pipe(icond_reg);
8892 %}
8893
8894 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8895 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8896
8897 ins_cost(INSN_COST * 2);
8898 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8899
8900 ins_encode %{
8901 __ cselw(as_Register($dst$$reg),
8902 as_Register($src$$reg),
8903 zr,
8904 (Assembler::Condition)$cmp$$cmpcode);
8905 %}
8906
8907 ins_pipe(icond_reg);
8908 %}
8909
8910 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8911 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8912
8913 ins_cost(INSN_COST * 2);
8914 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8915
8916 ins_encode %{
8917 __ cselw(as_Register($dst$$reg),
8918 as_Register($src$$reg),
8919 zr,
8920 (Assembler::Condition)$cmp$$cmpcode);
8921 %}
8922
8923 ins_pipe(icond_reg);
8924 %}
8925
8926 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8927 %{
8928 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8929
8930 ins_cost(INSN_COST * 3);
8931
8932 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8933 ins_encode %{
8934 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8935 __ fcsels(as_FloatRegister($dst$$reg),
8936 as_FloatRegister($src2$$reg),
8937 as_FloatRegister($src1$$reg),
8938 cond);
8939 %}
8940
8941 ins_pipe(fp_cond_reg_reg_s);
8942 %}
8943
8944 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8945 %{
8946 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8947
8948 ins_cost(INSN_COST * 3);
8949
8950 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8951 ins_encode %{
8952 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8953 __ fcsels(as_FloatRegister($dst$$reg),
8954 as_FloatRegister($src2$$reg),
8955 as_FloatRegister($src1$$reg),
8956 cond);
8957 %}
8958
8959 ins_pipe(fp_cond_reg_reg_s);
8960 %}
8961
8962 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8963 %{
8964 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8965
8966 ins_cost(INSN_COST * 3);
8967
8968 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8969 ins_encode %{
8970 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8971 __ fcseld(as_FloatRegister($dst$$reg),
8972 as_FloatRegister($src2$$reg),
8973 as_FloatRegister($src1$$reg),
8974 cond);
8975 %}
8976
8977 ins_pipe(fp_cond_reg_reg_d);
8978 %}
8979
8980 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8981 %{
8982 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8983
8984 ins_cost(INSN_COST * 3);
8985
8986 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8987 ins_encode %{
8988 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8989 __ fcseld(as_FloatRegister($dst$$reg),
8990 as_FloatRegister($src2$$reg),
8991 as_FloatRegister($src1$$reg),
8992 cond);
8993 %}
8994
8995 ins_pipe(fp_cond_reg_reg_d);
8996 %}
8997
8998 // ============================================================================
8999 // Arithmetic Instructions
9000 //
9001
9002 // Integer Addition
9003
9004 // TODO
9005 // these currently employ operations which do not set CR and hence are
9006 // not flagged as killing CR but we would like to isolate the cases
9007 // where we want to set flags from those where we don't. need to work
9008 // out how to do that.
9009
9010 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9011 match(Set dst (AddI src1 src2));
9012
9013 ins_cost(INSN_COST);
9014 format %{ "addw $dst, $src1, $src2" %}
9015
9016 ins_encode %{
9017 __ addw(as_Register($dst$$reg),
9018 as_Register($src1$$reg),
9019 as_Register($src2$$reg));
9020 %}
9021
9022 ins_pipe(ialu_reg_reg);
9023 %}
9024
9025 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9026 match(Set dst (AddI src1 src2));
9027
9028 ins_cost(INSN_COST);
9029 format %{ "addw $dst, $src1, $src2" %}
9030
9031 // use opcode to indicate that this is an add not a sub
9032 opcode(0x0);
9033
9034 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9035
9036 ins_pipe(ialu_reg_imm);
9037 %}
9038
9039 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9040 match(Set dst (AddI (ConvL2I src1) src2));
9041
9042 ins_cost(INSN_COST);
9043 format %{ "addw $dst, $src1, $src2" %}
9044
9045 // use opcode to indicate that this is an add not a sub
9046 opcode(0x0);
9047
9048 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9049
9050 ins_pipe(ialu_reg_imm);
9051 %}
9052
9053 // Pointer Addition
9054 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9055 match(Set dst (AddP src1 src2));
9056
9057 ins_cost(INSN_COST);
9058 format %{ "add $dst, $src1, $src2\t# ptr" %}
9059
9060 ins_encode %{
9061 __ add(as_Register($dst$$reg),
9062 as_Register($src1$$reg),
9063 as_Register($src2$$reg));
9064 %}
9065
9066 ins_pipe(ialu_reg_reg);
9067 %}
9068
9069 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9070 match(Set dst (AddP src1 (ConvI2L src2)));
9071
9072 ins_cost(1.9 * INSN_COST);
9073 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9074
9075 ins_encode %{
9076 __ add(as_Register($dst$$reg),
9077 as_Register($src1$$reg),
9078 as_Register($src2$$reg), ext::sxtw);
9079 %}
9080
9081 ins_pipe(ialu_reg_reg);
9082 %}
9083
9084 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9085 match(Set dst (AddP src1 (LShiftL src2 scale)));
9086
9087 ins_cost(1.9 * INSN_COST);
9088 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9089
9090 ins_encode %{
9091 __ lea(as_Register($dst$$reg),
9092 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9093 Address::lsl($scale$$constant)));
9094 %}
9095
9096 ins_pipe(ialu_reg_reg_shift);
9097 %}
9098
9099 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9100 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9101
9102 ins_cost(1.9 * INSN_COST);
9103 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9104
9105 ins_encode %{
9106 __ lea(as_Register($dst$$reg),
9107 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9108 Address::sxtw($scale$$constant)));
9109 %}
9110
9111 ins_pipe(ialu_reg_reg_shift);
9112 %}
9113
9114 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9115 match(Set dst (LShiftL (ConvI2L src) scale));
9116
9117 ins_cost(INSN_COST);
9118 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9119
9120 ins_encode %{
9121 __ sbfiz(as_Register($dst$$reg),
9122 as_Register($src$$reg),
9123 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9124 %}
9125
9126 ins_pipe(ialu_reg_shift);
9127 %}
9128
9129 // Pointer Immediate Addition
9130 // n.b. this needs to be more expensive than using an indirect memory
9131 // operand
9132 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9133 match(Set dst (AddP src1 src2));
9134
9135 ins_cost(INSN_COST);
9136 format %{ "add $dst, $src1, $src2\t# ptr" %}
9137
9138 // use opcode to indicate that this is an add not a sub
9139 opcode(0x0);
9140
9141 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9142
9143 ins_pipe(ialu_reg_imm);
9144 %}
9145
9146 // Long Addition
9147 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9148
9149 match(Set dst (AddL src1 src2));
9150
9151 ins_cost(INSN_COST);
9152 format %{ "add $dst, $src1, $src2" %}
9153
9154 ins_encode %{
9155 __ add(as_Register($dst$$reg),
9156 as_Register($src1$$reg),
9157 as_Register($src2$$reg));
9158 %}
9159
9160 ins_pipe(ialu_reg_reg);
9161 %}
9162
9163 // No constant pool entries requiredLong Immediate Addition.
9164 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9165 match(Set dst (AddL src1 src2));
9166
9167 ins_cost(INSN_COST);
9168 format %{ "add $dst, $src1, $src2" %}
9169
9170 // use opcode to indicate that this is an add not a sub
9171 opcode(0x0);
9172
9173 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9174
9175 ins_pipe(ialu_reg_imm);
9176 %}
9177
9178 // Integer Subtraction
9179 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9180 match(Set dst (SubI src1 src2));
9181
9182 ins_cost(INSN_COST);
9183 format %{ "subw $dst, $src1, $src2" %}
9184
9185 ins_encode %{
9186 __ subw(as_Register($dst$$reg),
9187 as_Register($src1$$reg),
9188 as_Register($src2$$reg));
9189 %}
9190
9191 ins_pipe(ialu_reg_reg);
9192 %}
9193
9194 // Immediate Subtraction
9195 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9196 match(Set dst (SubI src1 src2));
9197
9198 ins_cost(INSN_COST);
9199 format %{ "subw $dst, $src1, $src2" %}
9200
9201 // use opcode to indicate that this is a sub not an add
9202 opcode(0x1);
9203
9204 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9205
9206 ins_pipe(ialu_reg_imm);
9207 %}
9208
9209 // Long Subtraction
9210 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9211
9212 match(Set dst (SubL src1 src2));
9213
9214 ins_cost(INSN_COST);
9215 format %{ "sub $dst, $src1, $src2" %}
9216
9217 ins_encode %{
9218 __ sub(as_Register($dst$$reg),
9219 as_Register($src1$$reg),
9220 as_Register($src2$$reg));
9221 %}
9222
9223 ins_pipe(ialu_reg_reg);
9224 %}
9225
9226 // No constant pool entries requiredLong Immediate Subtraction.
9227 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9228 match(Set dst (SubL src1 src2));
9229
9230 ins_cost(INSN_COST);
9231 format %{ "sub$dst, $src1, $src2" %}
9232
9233 // use opcode to indicate that this is a sub not an add
9234 opcode(0x1);
9235
9236 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9237
9238 ins_pipe(ialu_reg_imm);
9239 %}
9240
9241 // Integer Negation (special case for sub)
9242
9243 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9244 match(Set dst (SubI zero src));
9245
9246 ins_cost(INSN_COST);
9247 format %{ "negw $dst, $src\t# int" %}
9248
9249 ins_encode %{
9250 __ negw(as_Register($dst$$reg),
9251 as_Register($src$$reg));
9252 %}
9253
9254 ins_pipe(ialu_reg);
9255 %}
9256
9257 // Long Negation
9258
9259 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9260 match(Set dst (SubL zero src));
9261
9262 ins_cost(INSN_COST);
9263 format %{ "neg $dst, $src\t# long" %}
9264
9265 ins_encode %{
9266 __ neg(as_Register($dst$$reg),
9267 as_Register($src$$reg));
9268 %}
9269
9270 ins_pipe(ialu_reg);
9271 %}
9272
9273 // Integer Multiply
9274
9275 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9276 match(Set dst (MulI src1 src2));
9277
9278 ins_cost(INSN_COST * 3);
9279 format %{ "mulw $dst, $src1, $src2" %}
9280
9281 ins_encode %{
9282 __ mulw(as_Register($dst$$reg),
9283 as_Register($src1$$reg),
9284 as_Register($src2$$reg));
9285 %}
9286
9287 ins_pipe(imul_reg_reg);
9288 %}
9289
9290 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9291 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9292
9293 ins_cost(INSN_COST * 3);
9294 format %{ "smull $dst, $src1, $src2" %}
9295
9296 ins_encode %{
9297 __ smull(as_Register($dst$$reg),
9298 as_Register($src1$$reg),
9299 as_Register($src2$$reg));
9300 %}
9301
9302 ins_pipe(imul_reg_reg);
9303 %}
9304
9305 // Long Multiply
9306
9307 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9308 match(Set dst (MulL src1 src2));
9309
9310 ins_cost(INSN_COST * 5);
9311 format %{ "mul $dst, $src1, $src2" %}
9312
9313 ins_encode %{
9314 __ mul(as_Register($dst$$reg),
9315 as_Register($src1$$reg),
9316 as_Register($src2$$reg));
9317 %}
9318
9319 ins_pipe(lmul_reg_reg);
9320 %}
9321
9322 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9323 %{
9324 match(Set dst (MulHiL src1 src2));
9325
9326 ins_cost(INSN_COST * 7);
9327 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9328
9329 ins_encode %{
9330 __ smulh(as_Register($dst$$reg),
9331 as_Register($src1$$reg),
9332 as_Register($src2$$reg));
9333 %}
9334
9335 ins_pipe(lmul_reg_reg);
9336 %}
9337
9338 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9339 %{
9340 match(Set dst (UMulHiL src1 src2));
9341
9342 ins_cost(INSN_COST * 7);
9343 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9344
9345 ins_encode %{
9346 __ umulh(as_Register($dst$$reg),
9347 as_Register($src1$$reg),
9348 as_Register($src2$$reg));
9349 %}
9350
9351 ins_pipe(lmul_reg_reg);
9352 %}
9353
9354 // Combined Integer Multiply & Add/Sub
9355
9356 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9357 match(Set dst (AddI src3 (MulI src1 src2)));
9358
9359 ins_cost(INSN_COST * 3);
9360 format %{ "madd $dst, $src1, $src2, $src3" %}
9361
9362 ins_encode %{
9363 __ maddw(as_Register($dst$$reg),
9364 as_Register($src1$$reg),
9365 as_Register($src2$$reg),
9366 as_Register($src3$$reg));
9367 %}
9368
9369 ins_pipe(imac_reg_reg);
9370 %}
9371
9372 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9373 match(Set dst (SubI src3 (MulI src1 src2)));
9374
9375 ins_cost(INSN_COST * 3);
9376 format %{ "msub $dst, $src1, $src2, $src3" %}
9377
9378 ins_encode %{
9379 __ msubw(as_Register($dst$$reg),
9380 as_Register($src1$$reg),
9381 as_Register($src2$$reg),
9382 as_Register($src3$$reg));
9383 %}
9384
9385 ins_pipe(imac_reg_reg);
9386 %}
9387
9388 // Combined Integer Multiply & Neg
9389
9390 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9391 match(Set dst (MulI (SubI zero src1) src2));
9392
9393 ins_cost(INSN_COST * 3);
9394 format %{ "mneg $dst, $src1, $src2" %}
9395
9396 ins_encode %{
9397 __ mnegw(as_Register($dst$$reg),
9398 as_Register($src1$$reg),
9399 as_Register($src2$$reg));
9400 %}
9401
9402 ins_pipe(imac_reg_reg);
9403 %}
9404
9405 // Combined Long Multiply & Add/Sub
9406
9407 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9408 match(Set dst (AddL src3 (MulL src1 src2)));
9409
9410 ins_cost(INSN_COST * 5);
9411 format %{ "madd $dst, $src1, $src2, $src3" %}
9412
9413 ins_encode %{
9414 __ madd(as_Register($dst$$reg),
9415 as_Register($src1$$reg),
9416 as_Register($src2$$reg),
9417 as_Register($src3$$reg));
9418 %}
9419
9420 ins_pipe(lmac_reg_reg);
9421 %}
9422
9423 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9424 match(Set dst (SubL src3 (MulL src1 src2)));
9425
9426 ins_cost(INSN_COST * 5);
9427 format %{ "msub $dst, $src1, $src2, $src3" %}
9428
9429 ins_encode %{
9430 __ msub(as_Register($dst$$reg),
9431 as_Register($src1$$reg),
9432 as_Register($src2$$reg),
9433 as_Register($src3$$reg));
9434 %}
9435
9436 ins_pipe(lmac_reg_reg);
9437 %}
9438
9439 // Combined Long Multiply & Neg
9440
9441 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9442 match(Set dst (MulL (SubL zero src1) src2));
9443
9444 ins_cost(INSN_COST * 5);
9445 format %{ "mneg $dst, $src1, $src2" %}
9446
9447 ins_encode %{
9448 __ mneg(as_Register($dst$$reg),
9449 as_Register($src1$$reg),
9450 as_Register($src2$$reg));
9451 %}
9452
9453 ins_pipe(lmac_reg_reg);
9454 %}
9455
9456 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9457
9458 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9459 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9460
9461 ins_cost(INSN_COST * 3);
9462 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9463
9464 ins_encode %{
9465 __ smaddl(as_Register($dst$$reg),
9466 as_Register($src1$$reg),
9467 as_Register($src2$$reg),
9468 as_Register($src3$$reg));
9469 %}
9470
9471 ins_pipe(imac_reg_reg);
9472 %}
9473
9474 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9475 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9476
9477 ins_cost(INSN_COST * 3);
9478 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9479
9480 ins_encode %{
9481 __ smsubl(as_Register($dst$$reg),
9482 as_Register($src1$$reg),
9483 as_Register($src2$$reg),
9484 as_Register($src3$$reg));
9485 %}
9486
9487 ins_pipe(imac_reg_reg);
9488 %}
9489
9490 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9491 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9492
9493 ins_cost(INSN_COST * 3);
9494 format %{ "smnegl $dst, $src1, $src2" %}
9495
9496 ins_encode %{
9497 __ smnegl(as_Register($dst$$reg),
9498 as_Register($src1$$reg),
9499 as_Register($src2$$reg));
9500 %}
9501
9502 ins_pipe(imac_reg_reg);
9503 %}
9504
9505 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9506
9507 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9508 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9509
9510 ins_cost(INSN_COST * 5);
9511 format %{ "mulw rscratch1, $src1, $src2\n\t"
9512 "maddw $dst, $src3, $src4, rscratch1" %}
9513
9514 ins_encode %{
9515 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9516 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9517
9518 ins_pipe(imac_reg_reg);
9519 %}
9520
9521 // Integer Divide
9522
9523 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9524 match(Set dst (DivI src1 src2));
9525
9526 ins_cost(INSN_COST * 19);
9527 format %{ "sdivw $dst, $src1, $src2" %}
9528
9529 ins_encode(aarch64_enc_divw(dst, src1, src2));
9530 ins_pipe(idiv_reg_reg);
9531 %}
9532
9533 // Long Divide
9534
9535 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9536 match(Set dst (DivL src1 src2));
9537
9538 ins_cost(INSN_COST * 35);
9539 format %{ "sdiv $dst, $src1, $src2" %}
9540
9541 ins_encode(aarch64_enc_div(dst, src1, src2));
9542 ins_pipe(ldiv_reg_reg);
9543 %}
9544
9545 // Integer Remainder
9546
9547 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9548 match(Set dst (ModI src1 src2));
9549
9550 ins_cost(INSN_COST * 22);
9551 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9552 "msubw $dst, rscratch1, $src2, $src1" %}
9553
9554 ins_encode(aarch64_enc_modw(dst, src1, src2));
9555 ins_pipe(idiv_reg_reg);
9556 %}
9557
9558 // Long Remainder
9559
9560 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9561 match(Set dst (ModL src1 src2));
9562
9563 ins_cost(INSN_COST * 38);
9564 format %{ "sdiv rscratch1, $src1, $src2\n"
9565 "msub $dst, rscratch1, $src2, $src1" %}
9566
9567 ins_encode(aarch64_enc_mod(dst, src1, src2));
9568 ins_pipe(ldiv_reg_reg);
9569 %}
9570
9571 // Unsigned Integer Divide
9572
9573 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9574 match(Set dst (UDivI src1 src2));
9575
9576 ins_cost(INSN_COST * 19);
9577 format %{ "udivw $dst, $src1, $src2" %}
9578
9579 ins_encode %{
9580 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9581 %}
9582
9583 ins_pipe(idiv_reg_reg);
9584 %}
9585
9586 // Unsigned Long Divide
9587
9588 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9589 match(Set dst (UDivL src1 src2));
9590
9591 ins_cost(INSN_COST * 35);
9592 format %{ "udiv $dst, $src1, $src2" %}
9593
9594 ins_encode %{
9595 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9596 %}
9597
9598 ins_pipe(ldiv_reg_reg);
9599 %}
9600
9601 // Unsigned Integer Remainder
9602
9603 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9604 match(Set dst (UModI src1 src2));
9605
9606 ins_cost(INSN_COST * 22);
9607 format %{ "udivw rscratch1, $src1, $src2\n\t"
9608 "msubw $dst, rscratch1, $src2, $src1" %}
9609
9610 ins_encode %{
9611 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9612 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9613 %}
9614
9615 ins_pipe(idiv_reg_reg);
9616 %}
9617
9618 // Unsigned Long Remainder
9619
9620 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9621 match(Set dst (UModL src1 src2));
9622
9623 ins_cost(INSN_COST * 38);
9624 format %{ "udiv rscratch1, $src1, $src2\n"
9625 "msub $dst, rscratch1, $src2, $src1" %}
9626
9627 ins_encode %{
9628 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9629 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9630 %}
9631
9632 ins_pipe(ldiv_reg_reg);
9633 %}
9634
9635 // Integer Shifts
9636
9637 // Shift Left Register
9638 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9639 match(Set dst (LShiftI src1 src2));
9640
9641 ins_cost(INSN_COST * 2);
9642 format %{ "lslvw $dst, $src1, $src2" %}
9643
9644 ins_encode %{
9645 __ lslvw(as_Register($dst$$reg),
9646 as_Register($src1$$reg),
9647 as_Register($src2$$reg));
9648 %}
9649
9650 ins_pipe(ialu_reg_reg_vshift);
9651 %}
9652
9653 // Shift Left Immediate
9654 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9655 match(Set dst (LShiftI src1 src2));
9656
9657 ins_cost(INSN_COST);
9658 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9659
9660 ins_encode %{
9661 __ lslw(as_Register($dst$$reg),
9662 as_Register($src1$$reg),
9663 $src2$$constant & 0x1f);
9664 %}
9665
9666 ins_pipe(ialu_reg_shift);
9667 %}
9668
9669 // Shift Right Logical Register
9670 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9671 match(Set dst (URShiftI src1 src2));
9672
9673 ins_cost(INSN_COST * 2);
9674 format %{ "lsrvw $dst, $src1, $src2" %}
9675
9676 ins_encode %{
9677 __ lsrvw(as_Register($dst$$reg),
9678 as_Register($src1$$reg),
9679 as_Register($src2$$reg));
9680 %}
9681
9682 ins_pipe(ialu_reg_reg_vshift);
9683 %}
9684
9685 // Shift Right Logical Immediate
9686 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9687 match(Set dst (URShiftI src1 src2));
9688
9689 ins_cost(INSN_COST);
9690 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9691
9692 ins_encode %{
9693 __ lsrw(as_Register($dst$$reg),
9694 as_Register($src1$$reg),
9695 $src2$$constant & 0x1f);
9696 %}
9697
9698 ins_pipe(ialu_reg_shift);
9699 %}
9700
9701 // Shift Right Arithmetic Register
9702 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9703 match(Set dst (RShiftI src1 src2));
9704
9705 ins_cost(INSN_COST * 2);
9706 format %{ "asrvw $dst, $src1, $src2" %}
9707
9708 ins_encode %{
9709 __ asrvw(as_Register($dst$$reg),
9710 as_Register($src1$$reg),
9711 as_Register($src2$$reg));
9712 %}
9713
9714 ins_pipe(ialu_reg_reg_vshift);
9715 %}
9716
9717 // Shift Right Arithmetic Immediate
9718 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9719 match(Set dst (RShiftI src1 src2));
9720
9721 ins_cost(INSN_COST);
9722 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9723
9724 ins_encode %{
9725 __ asrw(as_Register($dst$$reg),
9726 as_Register($src1$$reg),
9727 $src2$$constant & 0x1f);
9728 %}
9729
9730 ins_pipe(ialu_reg_shift);
9731 %}
9732
9733 // Combined Int Mask and Right Shift (using UBFM)
9734 // TODO
9735
9736 // Long Shifts
9737
9738 // Shift Left Register
9739 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9740 match(Set dst (LShiftL src1 src2));
9741
9742 ins_cost(INSN_COST * 2);
9743 format %{ "lslv $dst, $src1, $src2" %}
9744
9745 ins_encode %{
9746 __ lslv(as_Register($dst$$reg),
9747 as_Register($src1$$reg),
9748 as_Register($src2$$reg));
9749 %}
9750
9751 ins_pipe(ialu_reg_reg_vshift);
9752 %}
9753
9754 // Shift Left Immediate
9755 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9756 match(Set dst (LShiftL src1 src2));
9757
9758 ins_cost(INSN_COST);
9759 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9760
9761 ins_encode %{
9762 __ lsl(as_Register($dst$$reg),
9763 as_Register($src1$$reg),
9764 $src2$$constant & 0x3f);
9765 %}
9766
9767 ins_pipe(ialu_reg_shift);
9768 %}
9769
9770 // Shift Right Logical Register
9771 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9772 match(Set dst (URShiftL src1 src2));
9773
9774 ins_cost(INSN_COST * 2);
9775 format %{ "lsrv $dst, $src1, $src2" %}
9776
9777 ins_encode %{
9778 __ lsrv(as_Register($dst$$reg),
9779 as_Register($src1$$reg),
9780 as_Register($src2$$reg));
9781 %}
9782
9783 ins_pipe(ialu_reg_reg_vshift);
9784 %}
9785
9786 // Shift Right Logical Immediate
9787 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9788 match(Set dst (URShiftL src1 src2));
9789
9790 ins_cost(INSN_COST);
9791 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9792
9793 ins_encode %{
9794 __ lsr(as_Register($dst$$reg),
9795 as_Register($src1$$reg),
9796 $src2$$constant & 0x3f);
9797 %}
9798
9799 ins_pipe(ialu_reg_shift);
9800 %}
9801
9802 // A special-case pattern for card table stores.
9803 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9804 match(Set dst (URShiftL (CastP2X src1) src2));
9805
9806 ins_cost(INSN_COST);
9807 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9808
9809 ins_encode %{
9810 __ lsr(as_Register($dst$$reg),
9811 as_Register($src1$$reg),
9812 $src2$$constant & 0x3f);
9813 %}
9814
9815 ins_pipe(ialu_reg_shift);
9816 %}
9817
9818 // Shift Right Arithmetic Register
9819 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9820 match(Set dst (RShiftL src1 src2));
9821
9822 ins_cost(INSN_COST * 2);
9823 format %{ "asrv $dst, $src1, $src2" %}
9824
9825 ins_encode %{
9826 __ asrv(as_Register($dst$$reg),
9827 as_Register($src1$$reg),
9828 as_Register($src2$$reg));
9829 %}
9830
9831 ins_pipe(ialu_reg_reg_vshift);
9832 %}
9833
9834 // Shift Right Arithmetic Immediate
9835 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9836 match(Set dst (RShiftL src1 src2));
9837
9838 ins_cost(INSN_COST);
9839 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9840
9841 ins_encode %{
9842 __ asr(as_Register($dst$$reg),
9843 as_Register($src1$$reg),
9844 $src2$$constant & 0x3f);
9845 %}
9846
9847 ins_pipe(ialu_reg_shift);
9848 %}
9849
9850 // BEGIN This section of the file is automatically generated. Do not edit --------------
9851 // This section is generated from aarch64_ad.m4
9852
9853 // This pattern is automatically generated from aarch64_ad.m4
9854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9855 instruct regL_not_reg(iRegLNoSp dst,
9856 iRegL src1, immL_M1 m1,
9857 rFlagsReg cr) %{
9858 match(Set dst (XorL src1 m1));
9859 ins_cost(INSN_COST);
9860 format %{ "eon $dst, $src1, zr" %}
9861
9862 ins_encode %{
9863 __ eon(as_Register($dst$$reg),
9864 as_Register($src1$$reg),
9865 zr,
9866 Assembler::LSL, 0);
9867 %}
9868
9869 ins_pipe(ialu_reg);
9870 %}
9871
9872 // This pattern is automatically generated from aarch64_ad.m4
9873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9874 instruct regI_not_reg(iRegINoSp dst,
9875 iRegIorL2I src1, immI_M1 m1,
9876 rFlagsReg cr) %{
9877 match(Set dst (XorI src1 m1));
9878 ins_cost(INSN_COST);
9879 format %{ "eonw $dst, $src1, zr" %}
9880
9881 ins_encode %{
9882 __ eonw(as_Register($dst$$reg),
9883 as_Register($src1$$reg),
9884 zr,
9885 Assembler::LSL, 0);
9886 %}
9887
9888 ins_pipe(ialu_reg);
9889 %}
9890
9891 // This pattern is automatically generated from aarch64_ad.m4
9892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9893 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9894 immI0 zero, iRegIorL2I src1, immI src2) %{
9895 match(Set dst (SubI zero (URShiftI src1 src2)));
9896
9897 ins_cost(1.9 * INSN_COST);
9898 format %{ "negw $dst, $src1, LSR $src2" %}
9899
9900 ins_encode %{
9901 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9902 Assembler::LSR, $src2$$constant & 0x1f);
9903 %}
9904
9905 ins_pipe(ialu_reg_shift);
9906 %}
9907
9908 // This pattern is automatically generated from aarch64_ad.m4
9909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9910 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9911 immI0 zero, iRegIorL2I src1, immI src2) %{
9912 match(Set dst (SubI zero (RShiftI src1 src2)));
9913
9914 ins_cost(1.9 * INSN_COST);
9915 format %{ "negw $dst, $src1, ASR $src2" %}
9916
9917 ins_encode %{
9918 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9919 Assembler::ASR, $src2$$constant & 0x1f);
9920 %}
9921
9922 ins_pipe(ialu_reg_shift);
9923 %}
9924
9925 // This pattern is automatically generated from aarch64_ad.m4
9926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9927 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9928 immI0 zero, iRegIorL2I src1, immI src2) %{
9929 match(Set dst (SubI zero (LShiftI src1 src2)));
9930
9931 ins_cost(1.9 * INSN_COST);
9932 format %{ "negw $dst, $src1, LSL $src2" %}
9933
9934 ins_encode %{
9935 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9936 Assembler::LSL, $src2$$constant & 0x1f);
9937 %}
9938
9939 ins_pipe(ialu_reg_shift);
9940 %}
9941
9942 // This pattern is automatically generated from aarch64_ad.m4
9943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9944 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9945 immL0 zero, iRegL src1, immI src2) %{
9946 match(Set dst (SubL zero (URShiftL src1 src2)));
9947
9948 ins_cost(1.9 * INSN_COST);
9949 format %{ "neg $dst, $src1, LSR $src2" %}
9950
9951 ins_encode %{
9952 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9953 Assembler::LSR, $src2$$constant & 0x3f);
9954 %}
9955
9956 ins_pipe(ialu_reg_shift);
9957 %}
9958
9959 // This pattern is automatically generated from aarch64_ad.m4
9960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9961 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9962 immL0 zero, iRegL src1, immI src2) %{
9963 match(Set dst (SubL zero (RShiftL src1 src2)));
9964
9965 ins_cost(1.9 * INSN_COST);
9966 format %{ "neg $dst, $src1, ASR $src2" %}
9967
9968 ins_encode %{
9969 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9970 Assembler::ASR, $src2$$constant & 0x3f);
9971 %}
9972
9973 ins_pipe(ialu_reg_shift);
9974 %}
9975
9976 // This pattern is automatically generated from aarch64_ad.m4
9977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9978 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9979 immL0 zero, iRegL src1, immI src2) %{
9980 match(Set dst (SubL zero (LShiftL src1 src2)));
9981
9982 ins_cost(1.9 * INSN_COST);
9983 format %{ "neg $dst, $src1, LSL $src2" %}
9984
9985 ins_encode %{
9986 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9987 Assembler::LSL, $src2$$constant & 0x3f);
9988 %}
9989
9990 ins_pipe(ialu_reg_shift);
9991 %}
9992
9993 // This pattern is automatically generated from aarch64_ad.m4
9994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9995 instruct AndI_reg_not_reg(iRegINoSp dst,
9996 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
9997 match(Set dst (AndI src1 (XorI src2 m1)));
9998 ins_cost(INSN_COST);
9999 format %{ "bicw $dst, $src1, $src2" %}
10000
10001 ins_encode %{
10002 __ bicw(as_Register($dst$$reg),
10003 as_Register($src1$$reg),
10004 as_Register($src2$$reg),
10005 Assembler::LSL, 0);
10006 %}
10007
10008 ins_pipe(ialu_reg_reg);
10009 %}
10010
10011 // This pattern is automatically generated from aarch64_ad.m4
10012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10013 instruct AndL_reg_not_reg(iRegLNoSp dst,
10014 iRegL src1, iRegL src2, immL_M1 m1) %{
10015 match(Set dst (AndL src1 (XorL src2 m1)));
10016 ins_cost(INSN_COST);
10017 format %{ "bic $dst, $src1, $src2" %}
10018
10019 ins_encode %{
10020 __ bic(as_Register($dst$$reg),
10021 as_Register($src1$$reg),
10022 as_Register($src2$$reg),
10023 Assembler::LSL, 0);
10024 %}
10025
10026 ins_pipe(ialu_reg_reg);
10027 %}
10028
10029 // This pattern is automatically generated from aarch64_ad.m4
10030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10031 instruct OrI_reg_not_reg(iRegINoSp dst,
10032 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10033 match(Set dst (OrI src1 (XorI src2 m1)));
10034 ins_cost(INSN_COST);
10035 format %{ "ornw $dst, $src1, $src2" %}
10036
10037 ins_encode %{
10038 __ ornw(as_Register($dst$$reg),
10039 as_Register($src1$$reg),
10040 as_Register($src2$$reg),
10041 Assembler::LSL, 0);
10042 %}
10043
10044 ins_pipe(ialu_reg_reg);
10045 %}
10046
10047 // This pattern is automatically generated from aarch64_ad.m4
10048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10049 instruct OrL_reg_not_reg(iRegLNoSp dst,
10050 iRegL src1, iRegL src2, immL_M1 m1) %{
10051 match(Set dst (OrL src1 (XorL src2 m1)));
10052 ins_cost(INSN_COST);
10053 format %{ "orn $dst, $src1, $src2" %}
10054
10055 ins_encode %{
10056 __ orn(as_Register($dst$$reg),
10057 as_Register($src1$$reg),
10058 as_Register($src2$$reg),
10059 Assembler::LSL, 0);
10060 %}
10061
10062 ins_pipe(ialu_reg_reg);
10063 %}
10064
10065 // This pattern is automatically generated from aarch64_ad.m4
10066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10067 instruct XorI_reg_not_reg(iRegINoSp dst,
10068 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10069 match(Set dst (XorI m1 (XorI src2 src1)));
10070 ins_cost(INSN_COST);
10071 format %{ "eonw $dst, $src1, $src2" %}
10072
10073 ins_encode %{
10074 __ eonw(as_Register($dst$$reg),
10075 as_Register($src1$$reg),
10076 as_Register($src2$$reg),
10077 Assembler::LSL, 0);
10078 %}
10079
10080 ins_pipe(ialu_reg_reg);
10081 %}
10082
10083 // This pattern is automatically generated from aarch64_ad.m4
10084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10085 instruct XorL_reg_not_reg(iRegLNoSp dst,
10086 iRegL src1, iRegL src2, immL_M1 m1) %{
10087 match(Set dst (XorL m1 (XorL src2 src1)));
10088 ins_cost(INSN_COST);
10089 format %{ "eon $dst, $src1, $src2" %}
10090
10091 ins_encode %{
10092 __ eon(as_Register($dst$$reg),
10093 as_Register($src1$$reg),
10094 as_Register($src2$$reg),
10095 Assembler::LSL, 0);
10096 %}
10097
10098 ins_pipe(ialu_reg_reg);
10099 %}
10100
10101 // This pattern is automatically generated from aarch64_ad.m4
10102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10103 // val & (-1 ^ (val >>> shift)) ==> bicw
10104 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10105 iRegIorL2I src1, iRegIorL2I src2,
10106 immI src3, immI_M1 src4) %{
10107 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10108 ins_cost(1.9 * INSN_COST);
10109 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10110
10111 ins_encode %{
10112 __ bicw(as_Register($dst$$reg),
10113 as_Register($src1$$reg),
10114 as_Register($src2$$reg),
10115 Assembler::LSR,
10116 $src3$$constant & 0x1f);
10117 %}
10118
10119 ins_pipe(ialu_reg_reg_shift);
10120 %}
10121
10122 // This pattern is automatically generated from aarch64_ad.m4
10123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10124 // val & (-1 ^ (val >>> shift)) ==> bic
10125 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10126 iRegL src1, iRegL src2,
10127 immI src3, immL_M1 src4) %{
10128 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10129 ins_cost(1.9 * INSN_COST);
10130 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10131
10132 ins_encode %{
10133 __ bic(as_Register($dst$$reg),
10134 as_Register($src1$$reg),
10135 as_Register($src2$$reg),
10136 Assembler::LSR,
10137 $src3$$constant & 0x3f);
10138 %}
10139
10140 ins_pipe(ialu_reg_reg_shift);
10141 %}
10142
10143 // This pattern is automatically generated from aarch64_ad.m4
10144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10145 // val & (-1 ^ (val >> shift)) ==> bicw
10146 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10147 iRegIorL2I src1, iRegIorL2I src2,
10148 immI src3, immI_M1 src4) %{
10149 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10150 ins_cost(1.9 * INSN_COST);
10151 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10152
10153 ins_encode %{
10154 __ bicw(as_Register($dst$$reg),
10155 as_Register($src1$$reg),
10156 as_Register($src2$$reg),
10157 Assembler::ASR,
10158 $src3$$constant & 0x1f);
10159 %}
10160
10161 ins_pipe(ialu_reg_reg_shift);
10162 %}
10163
10164 // This pattern is automatically generated from aarch64_ad.m4
10165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10166 // val & (-1 ^ (val >> shift)) ==> bic
10167 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10168 iRegL src1, iRegL src2,
10169 immI src3, immL_M1 src4) %{
10170 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10171 ins_cost(1.9 * INSN_COST);
10172 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10173
10174 ins_encode %{
10175 __ bic(as_Register($dst$$reg),
10176 as_Register($src1$$reg),
10177 as_Register($src2$$reg),
10178 Assembler::ASR,
10179 $src3$$constant & 0x3f);
10180 %}
10181
10182 ins_pipe(ialu_reg_reg_shift);
10183 %}
10184
10185 // This pattern is automatically generated from aarch64_ad.m4
10186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10187 // val & (-1 ^ (val ror shift)) ==> bicw
10188 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10189 iRegIorL2I src1, iRegIorL2I src2,
10190 immI src3, immI_M1 src4) %{
10191 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10192 ins_cost(1.9 * INSN_COST);
10193 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10194
10195 ins_encode %{
10196 __ bicw(as_Register($dst$$reg),
10197 as_Register($src1$$reg),
10198 as_Register($src2$$reg),
10199 Assembler::ROR,
10200 $src3$$constant & 0x1f);
10201 %}
10202
10203 ins_pipe(ialu_reg_reg_shift);
10204 %}
10205
10206 // This pattern is automatically generated from aarch64_ad.m4
10207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10208 // val & (-1 ^ (val ror shift)) ==> bic
10209 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10210 iRegL src1, iRegL src2,
10211 immI src3, immL_M1 src4) %{
10212 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10213 ins_cost(1.9 * INSN_COST);
10214 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10215
10216 ins_encode %{
10217 __ bic(as_Register($dst$$reg),
10218 as_Register($src1$$reg),
10219 as_Register($src2$$reg),
10220 Assembler::ROR,
10221 $src3$$constant & 0x3f);
10222 %}
10223
10224 ins_pipe(ialu_reg_reg_shift);
10225 %}
10226
10227 // This pattern is automatically generated from aarch64_ad.m4
10228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10229 // val & (-1 ^ (val << shift)) ==> bicw
10230 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10231 iRegIorL2I src1, iRegIorL2I src2,
10232 immI src3, immI_M1 src4) %{
10233 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10234 ins_cost(1.9 * INSN_COST);
10235 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10236
10237 ins_encode %{
10238 __ bicw(as_Register($dst$$reg),
10239 as_Register($src1$$reg),
10240 as_Register($src2$$reg),
10241 Assembler::LSL,
10242 $src3$$constant & 0x1f);
10243 %}
10244
10245 ins_pipe(ialu_reg_reg_shift);
10246 %}
10247
10248 // This pattern is automatically generated from aarch64_ad.m4
10249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10250 // val & (-1 ^ (val << shift)) ==> bic
10251 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10252 iRegL src1, iRegL src2,
10253 immI src3, immL_M1 src4) %{
10254 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10255 ins_cost(1.9 * INSN_COST);
10256 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10257
10258 ins_encode %{
10259 __ bic(as_Register($dst$$reg),
10260 as_Register($src1$$reg),
10261 as_Register($src2$$reg),
10262 Assembler::LSL,
10263 $src3$$constant & 0x3f);
10264 %}
10265
10266 ins_pipe(ialu_reg_reg_shift);
10267 %}
10268
10269 // This pattern is automatically generated from aarch64_ad.m4
10270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10271 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10272 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10273 iRegIorL2I src1, iRegIorL2I src2,
10274 immI src3, immI_M1 src4) %{
10275 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10276 ins_cost(1.9 * INSN_COST);
10277 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10278
10279 ins_encode %{
10280 __ eonw(as_Register($dst$$reg),
10281 as_Register($src1$$reg),
10282 as_Register($src2$$reg),
10283 Assembler::LSR,
10284 $src3$$constant & 0x1f);
10285 %}
10286
10287 ins_pipe(ialu_reg_reg_shift);
10288 %}
10289
10290 // This pattern is automatically generated from aarch64_ad.m4
10291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10292 // val ^ (-1 ^ (val >>> shift)) ==> eon
10293 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10294 iRegL src1, iRegL src2,
10295 immI src3, immL_M1 src4) %{
10296 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10297 ins_cost(1.9 * INSN_COST);
10298 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10299
10300 ins_encode %{
10301 __ eon(as_Register($dst$$reg),
10302 as_Register($src1$$reg),
10303 as_Register($src2$$reg),
10304 Assembler::LSR,
10305 $src3$$constant & 0x3f);
10306 %}
10307
10308 ins_pipe(ialu_reg_reg_shift);
10309 %}
10310
10311 // This pattern is automatically generated from aarch64_ad.m4
10312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10313 // val ^ (-1 ^ (val >> shift)) ==> eonw
10314 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10315 iRegIorL2I src1, iRegIorL2I src2,
10316 immI src3, immI_M1 src4) %{
10317 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10318 ins_cost(1.9 * INSN_COST);
10319 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10320
10321 ins_encode %{
10322 __ eonw(as_Register($dst$$reg),
10323 as_Register($src1$$reg),
10324 as_Register($src2$$reg),
10325 Assembler::ASR,
10326 $src3$$constant & 0x1f);
10327 %}
10328
10329 ins_pipe(ialu_reg_reg_shift);
10330 %}
10331
10332 // This pattern is automatically generated from aarch64_ad.m4
10333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10334 // val ^ (-1 ^ (val >> shift)) ==> eon
10335 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10336 iRegL src1, iRegL src2,
10337 immI src3, immL_M1 src4) %{
10338 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10339 ins_cost(1.9 * INSN_COST);
10340 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10341
10342 ins_encode %{
10343 __ eon(as_Register($dst$$reg),
10344 as_Register($src1$$reg),
10345 as_Register($src2$$reg),
10346 Assembler::ASR,
10347 $src3$$constant & 0x3f);
10348 %}
10349
10350 ins_pipe(ialu_reg_reg_shift);
10351 %}
10352
10353 // This pattern is automatically generated from aarch64_ad.m4
10354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10355 // val ^ (-1 ^ (val ror shift)) ==> eonw
10356 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10357 iRegIorL2I src1, iRegIorL2I src2,
10358 immI src3, immI_M1 src4) %{
10359 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10360 ins_cost(1.9 * INSN_COST);
10361 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10362
10363 ins_encode %{
10364 __ eonw(as_Register($dst$$reg),
10365 as_Register($src1$$reg),
10366 as_Register($src2$$reg),
10367 Assembler::ROR,
10368 $src3$$constant & 0x1f);
10369 %}
10370
10371 ins_pipe(ialu_reg_reg_shift);
10372 %}
10373
10374 // This pattern is automatically generated from aarch64_ad.m4
10375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10376 // val ^ (-1 ^ (val ror shift)) ==> eon
10377 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10378 iRegL src1, iRegL src2,
10379 immI src3, immL_M1 src4) %{
10380 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10381 ins_cost(1.9 * INSN_COST);
10382 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10383
10384 ins_encode %{
10385 __ eon(as_Register($dst$$reg),
10386 as_Register($src1$$reg),
10387 as_Register($src2$$reg),
10388 Assembler::ROR,
10389 $src3$$constant & 0x3f);
10390 %}
10391
10392 ins_pipe(ialu_reg_reg_shift);
10393 %}
10394
10395 // This pattern is automatically generated from aarch64_ad.m4
10396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10397 // val ^ (-1 ^ (val << shift)) ==> eonw
10398 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10399 iRegIorL2I src1, iRegIorL2I src2,
10400 immI src3, immI_M1 src4) %{
10401 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10402 ins_cost(1.9 * INSN_COST);
10403 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10404
10405 ins_encode %{
10406 __ eonw(as_Register($dst$$reg),
10407 as_Register($src1$$reg),
10408 as_Register($src2$$reg),
10409 Assembler::LSL,
10410 $src3$$constant & 0x1f);
10411 %}
10412
10413 ins_pipe(ialu_reg_reg_shift);
10414 %}
10415
10416 // This pattern is automatically generated from aarch64_ad.m4
10417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10418 // val ^ (-1 ^ (val << shift)) ==> eon
10419 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10420 iRegL src1, iRegL src2,
10421 immI src3, immL_M1 src4) %{
10422 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10423 ins_cost(1.9 * INSN_COST);
10424 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10425
10426 ins_encode %{
10427 __ eon(as_Register($dst$$reg),
10428 as_Register($src1$$reg),
10429 as_Register($src2$$reg),
10430 Assembler::LSL,
10431 $src3$$constant & 0x3f);
10432 %}
10433
10434 ins_pipe(ialu_reg_reg_shift);
10435 %}
10436
10437 // This pattern is automatically generated from aarch64_ad.m4
10438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10439 // val | (-1 ^ (val >>> shift)) ==> ornw
10440 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10441 iRegIorL2I src1, iRegIorL2I src2,
10442 immI src3, immI_M1 src4) %{
10443 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10444 ins_cost(1.9 * INSN_COST);
10445 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10446
10447 ins_encode %{
10448 __ ornw(as_Register($dst$$reg),
10449 as_Register($src1$$reg),
10450 as_Register($src2$$reg),
10451 Assembler::LSR,
10452 $src3$$constant & 0x1f);
10453 %}
10454
10455 ins_pipe(ialu_reg_reg_shift);
10456 %}
10457
10458 // This pattern is automatically generated from aarch64_ad.m4
10459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10460 // val | (-1 ^ (val >>> shift)) ==> orn
10461 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10462 iRegL src1, iRegL src2,
10463 immI src3, immL_M1 src4) %{
10464 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10465 ins_cost(1.9 * INSN_COST);
10466 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10467
10468 ins_encode %{
10469 __ orn(as_Register($dst$$reg),
10470 as_Register($src1$$reg),
10471 as_Register($src2$$reg),
10472 Assembler::LSR,
10473 $src3$$constant & 0x3f);
10474 %}
10475
10476 ins_pipe(ialu_reg_reg_shift);
10477 %}
10478
10479 // This pattern is automatically generated from aarch64_ad.m4
10480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10481 // val | (-1 ^ (val >> shift)) ==> ornw
10482 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10483 iRegIorL2I src1, iRegIorL2I src2,
10484 immI src3, immI_M1 src4) %{
10485 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10486 ins_cost(1.9 * INSN_COST);
10487 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10488
10489 ins_encode %{
10490 __ ornw(as_Register($dst$$reg),
10491 as_Register($src1$$reg),
10492 as_Register($src2$$reg),
10493 Assembler::ASR,
10494 $src3$$constant & 0x1f);
10495 %}
10496
10497 ins_pipe(ialu_reg_reg_shift);
10498 %}
10499
10500 // This pattern is automatically generated from aarch64_ad.m4
10501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10502 // val | (-1 ^ (val >> shift)) ==> orn
10503 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10504 iRegL src1, iRegL src2,
10505 immI src3, immL_M1 src4) %{
10506 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10507 ins_cost(1.9 * INSN_COST);
10508 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10509
10510 ins_encode %{
10511 __ orn(as_Register($dst$$reg),
10512 as_Register($src1$$reg),
10513 as_Register($src2$$reg),
10514 Assembler::ASR,
10515 $src3$$constant & 0x3f);
10516 %}
10517
10518 ins_pipe(ialu_reg_reg_shift);
10519 %}
10520
10521 // This pattern is automatically generated from aarch64_ad.m4
10522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10523 // val | (-1 ^ (val ror shift)) ==> ornw
10524 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10525 iRegIorL2I src1, iRegIorL2I src2,
10526 immI src3, immI_M1 src4) %{
10527 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10528 ins_cost(1.9 * INSN_COST);
10529 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10530
10531 ins_encode %{
10532 __ ornw(as_Register($dst$$reg),
10533 as_Register($src1$$reg),
10534 as_Register($src2$$reg),
10535 Assembler::ROR,
10536 $src3$$constant & 0x1f);
10537 %}
10538
10539 ins_pipe(ialu_reg_reg_shift);
10540 %}
10541
10542 // This pattern is automatically generated from aarch64_ad.m4
10543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10544 // val | (-1 ^ (val ror shift)) ==> orn
10545 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10546 iRegL src1, iRegL src2,
10547 immI src3, immL_M1 src4) %{
10548 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10549 ins_cost(1.9 * INSN_COST);
10550 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10551
10552 ins_encode %{
10553 __ orn(as_Register($dst$$reg),
10554 as_Register($src1$$reg),
10555 as_Register($src2$$reg),
10556 Assembler::ROR,
10557 $src3$$constant & 0x3f);
10558 %}
10559
10560 ins_pipe(ialu_reg_reg_shift);
10561 %}
10562
10563 // This pattern is automatically generated from aarch64_ad.m4
10564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10565 // val | (-1 ^ (val << shift)) ==> ornw
10566 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10567 iRegIorL2I src1, iRegIorL2I src2,
10568 immI src3, immI_M1 src4) %{
10569 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10570 ins_cost(1.9 * INSN_COST);
10571 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10572
10573 ins_encode %{
10574 __ ornw(as_Register($dst$$reg),
10575 as_Register($src1$$reg),
10576 as_Register($src2$$reg),
10577 Assembler::LSL,
10578 $src3$$constant & 0x1f);
10579 %}
10580
10581 ins_pipe(ialu_reg_reg_shift);
10582 %}
10583
10584 // This pattern is automatically generated from aarch64_ad.m4
10585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10586 // val | (-1 ^ (val << shift)) ==> orn
10587 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10588 iRegL src1, iRegL src2,
10589 immI src3, immL_M1 src4) %{
10590 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10591 ins_cost(1.9 * INSN_COST);
10592 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10593
10594 ins_encode %{
10595 __ orn(as_Register($dst$$reg),
10596 as_Register($src1$$reg),
10597 as_Register($src2$$reg),
10598 Assembler::LSL,
10599 $src3$$constant & 0x3f);
10600 %}
10601
10602 ins_pipe(ialu_reg_reg_shift);
10603 %}
10604
10605 // This pattern is automatically generated from aarch64_ad.m4
10606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10607 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10608 iRegIorL2I src1, iRegIorL2I src2,
10609 immI src3) %{
10610 match(Set dst (AndI src1 (URShiftI src2 src3)));
10611
10612 ins_cost(1.9 * INSN_COST);
10613 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10614
10615 ins_encode %{
10616 __ andw(as_Register($dst$$reg),
10617 as_Register($src1$$reg),
10618 as_Register($src2$$reg),
10619 Assembler::LSR,
10620 $src3$$constant & 0x1f);
10621 %}
10622
10623 ins_pipe(ialu_reg_reg_shift);
10624 %}
10625
10626 // This pattern is automatically generated from aarch64_ad.m4
10627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10628 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10629 iRegL src1, iRegL src2,
10630 immI src3) %{
10631 match(Set dst (AndL src1 (URShiftL src2 src3)));
10632
10633 ins_cost(1.9 * INSN_COST);
10634 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10635
10636 ins_encode %{
10637 __ andr(as_Register($dst$$reg),
10638 as_Register($src1$$reg),
10639 as_Register($src2$$reg),
10640 Assembler::LSR,
10641 $src3$$constant & 0x3f);
10642 %}
10643
10644 ins_pipe(ialu_reg_reg_shift);
10645 %}
10646
10647 // This pattern is automatically generated from aarch64_ad.m4
10648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10649 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10650 iRegIorL2I src1, iRegIorL2I src2,
10651 immI src3) %{
10652 match(Set dst (AndI src1 (RShiftI src2 src3)));
10653
10654 ins_cost(1.9 * INSN_COST);
10655 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10656
10657 ins_encode %{
10658 __ andw(as_Register($dst$$reg),
10659 as_Register($src1$$reg),
10660 as_Register($src2$$reg),
10661 Assembler::ASR,
10662 $src3$$constant & 0x1f);
10663 %}
10664
10665 ins_pipe(ialu_reg_reg_shift);
10666 %}
10667
10668 // This pattern is automatically generated from aarch64_ad.m4
10669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10670 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10671 iRegL src1, iRegL src2,
10672 immI src3) %{
10673 match(Set dst (AndL src1 (RShiftL src2 src3)));
10674
10675 ins_cost(1.9 * INSN_COST);
10676 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10677
10678 ins_encode %{
10679 __ andr(as_Register($dst$$reg),
10680 as_Register($src1$$reg),
10681 as_Register($src2$$reg),
10682 Assembler::ASR,
10683 $src3$$constant & 0x3f);
10684 %}
10685
10686 ins_pipe(ialu_reg_reg_shift);
10687 %}
10688
10689 // This pattern is automatically generated from aarch64_ad.m4
10690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10691 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10692 iRegIorL2I src1, iRegIorL2I src2,
10693 immI src3) %{
10694 match(Set dst (AndI src1 (LShiftI src2 src3)));
10695
10696 ins_cost(1.9 * INSN_COST);
10697 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10698
10699 ins_encode %{
10700 __ andw(as_Register($dst$$reg),
10701 as_Register($src1$$reg),
10702 as_Register($src2$$reg),
10703 Assembler::LSL,
10704 $src3$$constant & 0x1f);
10705 %}
10706
10707 ins_pipe(ialu_reg_reg_shift);
10708 %}
10709
10710 // This pattern is automatically generated from aarch64_ad.m4
10711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10712 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10713 iRegL src1, iRegL src2,
10714 immI src3) %{
10715 match(Set dst (AndL src1 (LShiftL src2 src3)));
10716
10717 ins_cost(1.9 * INSN_COST);
10718 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10719
10720 ins_encode %{
10721 __ andr(as_Register($dst$$reg),
10722 as_Register($src1$$reg),
10723 as_Register($src2$$reg),
10724 Assembler::LSL,
10725 $src3$$constant & 0x3f);
10726 %}
10727
10728 ins_pipe(ialu_reg_reg_shift);
10729 %}
10730
10731 // This pattern is automatically generated from aarch64_ad.m4
10732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10733 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10734 iRegIorL2I src1, iRegIorL2I src2,
10735 immI src3) %{
10736 match(Set dst (AndI src1 (RotateRight src2 src3)));
10737
10738 ins_cost(1.9 * INSN_COST);
10739 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10740
10741 ins_encode %{
10742 __ andw(as_Register($dst$$reg),
10743 as_Register($src1$$reg),
10744 as_Register($src2$$reg),
10745 Assembler::ROR,
10746 $src3$$constant & 0x1f);
10747 %}
10748
10749 ins_pipe(ialu_reg_reg_shift);
10750 %}
10751
10752 // This pattern is automatically generated from aarch64_ad.m4
10753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10754 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10755 iRegL src1, iRegL src2,
10756 immI src3) %{
10757 match(Set dst (AndL src1 (RotateRight src2 src3)));
10758
10759 ins_cost(1.9 * INSN_COST);
10760 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10761
10762 ins_encode %{
10763 __ andr(as_Register($dst$$reg),
10764 as_Register($src1$$reg),
10765 as_Register($src2$$reg),
10766 Assembler::ROR,
10767 $src3$$constant & 0x3f);
10768 %}
10769
10770 ins_pipe(ialu_reg_reg_shift);
10771 %}
10772
10773 // This pattern is automatically generated from aarch64_ad.m4
10774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10775 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10776 iRegIorL2I src1, iRegIorL2I src2,
10777 immI src3) %{
10778 match(Set dst (XorI src1 (URShiftI src2 src3)));
10779
10780 ins_cost(1.9 * INSN_COST);
10781 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10782
10783 ins_encode %{
10784 __ eorw(as_Register($dst$$reg),
10785 as_Register($src1$$reg),
10786 as_Register($src2$$reg),
10787 Assembler::LSR,
10788 $src3$$constant & 0x1f);
10789 %}
10790
10791 ins_pipe(ialu_reg_reg_shift);
10792 %}
10793
10794 // This pattern is automatically generated from aarch64_ad.m4
10795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10796 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10797 iRegL src1, iRegL src2,
10798 immI src3) %{
10799 match(Set dst (XorL src1 (URShiftL src2 src3)));
10800
10801 ins_cost(1.9 * INSN_COST);
10802 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10803
10804 ins_encode %{
10805 __ eor(as_Register($dst$$reg),
10806 as_Register($src1$$reg),
10807 as_Register($src2$$reg),
10808 Assembler::LSR,
10809 $src3$$constant & 0x3f);
10810 %}
10811
10812 ins_pipe(ialu_reg_reg_shift);
10813 %}
10814
10815 // This pattern is automatically generated from aarch64_ad.m4
10816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10817 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10818 iRegIorL2I src1, iRegIorL2I src2,
10819 immI src3) %{
10820 match(Set dst (XorI src1 (RShiftI src2 src3)));
10821
10822 ins_cost(1.9 * INSN_COST);
10823 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10824
10825 ins_encode %{
10826 __ eorw(as_Register($dst$$reg),
10827 as_Register($src1$$reg),
10828 as_Register($src2$$reg),
10829 Assembler::ASR,
10830 $src3$$constant & 0x1f);
10831 %}
10832
10833 ins_pipe(ialu_reg_reg_shift);
10834 %}
10835
10836 // This pattern is automatically generated from aarch64_ad.m4
10837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10838 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10839 iRegL src1, iRegL src2,
10840 immI src3) %{
10841 match(Set dst (XorL src1 (RShiftL src2 src3)));
10842
10843 ins_cost(1.9 * INSN_COST);
10844 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10845
10846 ins_encode %{
10847 __ eor(as_Register($dst$$reg),
10848 as_Register($src1$$reg),
10849 as_Register($src2$$reg),
10850 Assembler::ASR,
10851 $src3$$constant & 0x3f);
10852 %}
10853
10854 ins_pipe(ialu_reg_reg_shift);
10855 %}
10856
10857 // This pattern is automatically generated from aarch64_ad.m4
10858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10859 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10860 iRegIorL2I src1, iRegIorL2I src2,
10861 immI src3) %{
10862 match(Set dst (XorI src1 (LShiftI src2 src3)));
10863
10864 ins_cost(1.9 * INSN_COST);
10865 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10866
10867 ins_encode %{
10868 __ eorw(as_Register($dst$$reg),
10869 as_Register($src1$$reg),
10870 as_Register($src2$$reg),
10871 Assembler::LSL,
10872 $src3$$constant & 0x1f);
10873 %}
10874
10875 ins_pipe(ialu_reg_reg_shift);
10876 %}
10877
10878 // This pattern is automatically generated from aarch64_ad.m4
10879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10880 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10881 iRegL src1, iRegL src2,
10882 immI src3) %{
10883 match(Set dst (XorL src1 (LShiftL src2 src3)));
10884
10885 ins_cost(1.9 * INSN_COST);
10886 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10887
10888 ins_encode %{
10889 __ eor(as_Register($dst$$reg),
10890 as_Register($src1$$reg),
10891 as_Register($src2$$reg),
10892 Assembler::LSL,
10893 $src3$$constant & 0x3f);
10894 %}
10895
10896 ins_pipe(ialu_reg_reg_shift);
10897 %}
10898
10899 // This pattern is automatically generated from aarch64_ad.m4
10900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10901 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10902 iRegIorL2I src1, iRegIorL2I src2,
10903 immI src3) %{
10904 match(Set dst (XorI src1 (RotateRight src2 src3)));
10905
10906 ins_cost(1.9 * INSN_COST);
10907 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10908
10909 ins_encode %{
10910 __ eorw(as_Register($dst$$reg),
10911 as_Register($src1$$reg),
10912 as_Register($src2$$reg),
10913 Assembler::ROR,
10914 $src3$$constant & 0x1f);
10915 %}
10916
10917 ins_pipe(ialu_reg_reg_shift);
10918 %}
10919
10920 // This pattern is automatically generated from aarch64_ad.m4
10921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10922 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10923 iRegL src1, iRegL src2,
10924 immI src3) %{
10925 match(Set dst (XorL src1 (RotateRight src2 src3)));
10926
10927 ins_cost(1.9 * INSN_COST);
10928 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10929
10930 ins_encode %{
10931 __ eor(as_Register($dst$$reg),
10932 as_Register($src1$$reg),
10933 as_Register($src2$$reg),
10934 Assembler::ROR,
10935 $src3$$constant & 0x3f);
10936 %}
10937
10938 ins_pipe(ialu_reg_reg_shift);
10939 %}
10940
10941 // This pattern is automatically generated from aarch64_ad.m4
10942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10943 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10944 iRegIorL2I src1, iRegIorL2I src2,
10945 immI src3) %{
10946 match(Set dst (OrI src1 (URShiftI src2 src3)));
10947
10948 ins_cost(1.9 * INSN_COST);
10949 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10950
10951 ins_encode %{
10952 __ orrw(as_Register($dst$$reg),
10953 as_Register($src1$$reg),
10954 as_Register($src2$$reg),
10955 Assembler::LSR,
10956 $src3$$constant & 0x1f);
10957 %}
10958
10959 ins_pipe(ialu_reg_reg_shift);
10960 %}
10961
10962 // This pattern is automatically generated from aarch64_ad.m4
10963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10964 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10965 iRegL src1, iRegL src2,
10966 immI src3) %{
10967 match(Set dst (OrL src1 (URShiftL src2 src3)));
10968
10969 ins_cost(1.9 * INSN_COST);
10970 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10971
10972 ins_encode %{
10973 __ orr(as_Register($dst$$reg),
10974 as_Register($src1$$reg),
10975 as_Register($src2$$reg),
10976 Assembler::LSR,
10977 $src3$$constant & 0x3f);
10978 %}
10979
10980 ins_pipe(ialu_reg_reg_shift);
10981 %}
10982
10983 // This pattern is automatically generated from aarch64_ad.m4
10984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10985 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10986 iRegIorL2I src1, iRegIorL2I src2,
10987 immI src3) %{
10988 match(Set dst (OrI src1 (RShiftI src2 src3)));
10989
10990 ins_cost(1.9 * INSN_COST);
10991 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10992
10993 ins_encode %{
10994 __ orrw(as_Register($dst$$reg),
10995 as_Register($src1$$reg),
10996 as_Register($src2$$reg),
10997 Assembler::ASR,
10998 $src3$$constant & 0x1f);
10999 %}
11000
11001 ins_pipe(ialu_reg_reg_shift);
11002 %}
11003
11004 // This pattern is automatically generated from aarch64_ad.m4
11005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11006 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11007 iRegL src1, iRegL src2,
11008 immI src3) %{
11009 match(Set dst (OrL src1 (RShiftL src2 src3)));
11010
11011 ins_cost(1.9 * INSN_COST);
11012 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11013
11014 ins_encode %{
11015 __ orr(as_Register($dst$$reg),
11016 as_Register($src1$$reg),
11017 as_Register($src2$$reg),
11018 Assembler::ASR,
11019 $src3$$constant & 0x3f);
11020 %}
11021
11022 ins_pipe(ialu_reg_reg_shift);
11023 %}
11024
11025 // This pattern is automatically generated from aarch64_ad.m4
11026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11027 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11028 iRegIorL2I src1, iRegIorL2I src2,
11029 immI src3) %{
11030 match(Set dst (OrI src1 (LShiftI src2 src3)));
11031
11032 ins_cost(1.9 * INSN_COST);
11033 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11034
11035 ins_encode %{
11036 __ orrw(as_Register($dst$$reg),
11037 as_Register($src1$$reg),
11038 as_Register($src2$$reg),
11039 Assembler::LSL,
11040 $src3$$constant & 0x1f);
11041 %}
11042
11043 ins_pipe(ialu_reg_reg_shift);
11044 %}
11045
11046 // This pattern is automatically generated from aarch64_ad.m4
11047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11048 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11049 iRegL src1, iRegL src2,
11050 immI src3) %{
11051 match(Set dst (OrL src1 (LShiftL src2 src3)));
11052
11053 ins_cost(1.9 * INSN_COST);
11054 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11055
11056 ins_encode %{
11057 __ orr(as_Register($dst$$reg),
11058 as_Register($src1$$reg),
11059 as_Register($src2$$reg),
11060 Assembler::LSL,
11061 $src3$$constant & 0x3f);
11062 %}
11063
11064 ins_pipe(ialu_reg_reg_shift);
11065 %}
11066
11067 // This pattern is automatically generated from aarch64_ad.m4
11068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11069 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11070 iRegIorL2I src1, iRegIorL2I src2,
11071 immI src3) %{
11072 match(Set dst (OrI src1 (RotateRight src2 src3)));
11073
11074 ins_cost(1.9 * INSN_COST);
11075 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11076
11077 ins_encode %{
11078 __ orrw(as_Register($dst$$reg),
11079 as_Register($src1$$reg),
11080 as_Register($src2$$reg),
11081 Assembler::ROR,
11082 $src3$$constant & 0x1f);
11083 %}
11084
11085 ins_pipe(ialu_reg_reg_shift);
11086 %}
11087
11088 // This pattern is automatically generated from aarch64_ad.m4
11089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11090 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11091 iRegL src1, iRegL src2,
11092 immI src3) %{
11093 match(Set dst (OrL src1 (RotateRight src2 src3)));
11094
11095 ins_cost(1.9 * INSN_COST);
11096 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11097
11098 ins_encode %{
11099 __ orr(as_Register($dst$$reg),
11100 as_Register($src1$$reg),
11101 as_Register($src2$$reg),
11102 Assembler::ROR,
11103 $src3$$constant & 0x3f);
11104 %}
11105
11106 ins_pipe(ialu_reg_reg_shift);
11107 %}
11108
11109 // This pattern is automatically generated from aarch64_ad.m4
11110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11111 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11112 iRegIorL2I src1, iRegIorL2I src2,
11113 immI src3) %{
11114 match(Set dst (AddI src1 (URShiftI src2 src3)));
11115
11116 ins_cost(1.9 * INSN_COST);
11117 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11118
11119 ins_encode %{
11120 __ addw(as_Register($dst$$reg),
11121 as_Register($src1$$reg),
11122 as_Register($src2$$reg),
11123 Assembler::LSR,
11124 $src3$$constant & 0x1f);
11125 %}
11126
11127 ins_pipe(ialu_reg_reg_shift);
11128 %}
11129
11130 // This pattern is automatically generated from aarch64_ad.m4
11131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11132 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11133 iRegL src1, iRegL src2,
11134 immI src3) %{
11135 match(Set dst (AddL src1 (URShiftL src2 src3)));
11136
11137 ins_cost(1.9 * INSN_COST);
11138 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11139
11140 ins_encode %{
11141 __ add(as_Register($dst$$reg),
11142 as_Register($src1$$reg),
11143 as_Register($src2$$reg),
11144 Assembler::LSR,
11145 $src3$$constant & 0x3f);
11146 %}
11147
11148 ins_pipe(ialu_reg_reg_shift);
11149 %}
11150
11151 // This pattern is automatically generated from aarch64_ad.m4
11152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11153 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11154 iRegIorL2I src1, iRegIorL2I src2,
11155 immI src3) %{
11156 match(Set dst (AddI src1 (RShiftI src2 src3)));
11157
11158 ins_cost(1.9 * INSN_COST);
11159 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11160
11161 ins_encode %{
11162 __ addw(as_Register($dst$$reg),
11163 as_Register($src1$$reg),
11164 as_Register($src2$$reg),
11165 Assembler::ASR,
11166 $src3$$constant & 0x1f);
11167 %}
11168
11169 ins_pipe(ialu_reg_reg_shift);
11170 %}
11171
11172 // This pattern is automatically generated from aarch64_ad.m4
11173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11174 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11175 iRegL src1, iRegL src2,
11176 immI src3) %{
11177 match(Set dst (AddL src1 (RShiftL src2 src3)));
11178
11179 ins_cost(1.9 * INSN_COST);
11180 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11181
11182 ins_encode %{
11183 __ add(as_Register($dst$$reg),
11184 as_Register($src1$$reg),
11185 as_Register($src2$$reg),
11186 Assembler::ASR,
11187 $src3$$constant & 0x3f);
11188 %}
11189
11190 ins_pipe(ialu_reg_reg_shift);
11191 %}
11192
11193 // This pattern is automatically generated from aarch64_ad.m4
11194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11195 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11196 iRegIorL2I src1, iRegIorL2I src2,
11197 immI src3) %{
11198 match(Set dst (AddI src1 (LShiftI src2 src3)));
11199
11200 ins_cost(1.9 * INSN_COST);
11201 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11202
11203 ins_encode %{
11204 __ addw(as_Register($dst$$reg),
11205 as_Register($src1$$reg),
11206 as_Register($src2$$reg),
11207 Assembler::LSL,
11208 $src3$$constant & 0x1f);
11209 %}
11210
11211 ins_pipe(ialu_reg_reg_shift);
11212 %}
11213
11214 // This pattern is automatically generated from aarch64_ad.m4
11215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11216 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11217 iRegL src1, iRegL src2,
11218 immI src3) %{
11219 match(Set dst (AddL src1 (LShiftL src2 src3)));
11220
11221 ins_cost(1.9 * INSN_COST);
11222 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11223
11224 ins_encode %{
11225 __ add(as_Register($dst$$reg),
11226 as_Register($src1$$reg),
11227 as_Register($src2$$reg),
11228 Assembler::LSL,
11229 $src3$$constant & 0x3f);
11230 %}
11231
11232 ins_pipe(ialu_reg_reg_shift);
11233 %}
11234
11235 // This pattern is automatically generated from aarch64_ad.m4
11236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11237 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11238 iRegIorL2I src1, iRegIorL2I src2,
11239 immI src3) %{
11240 match(Set dst (SubI src1 (URShiftI src2 src3)));
11241
11242 ins_cost(1.9 * INSN_COST);
11243 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11244
11245 ins_encode %{
11246 __ subw(as_Register($dst$$reg),
11247 as_Register($src1$$reg),
11248 as_Register($src2$$reg),
11249 Assembler::LSR,
11250 $src3$$constant & 0x1f);
11251 %}
11252
11253 ins_pipe(ialu_reg_reg_shift);
11254 %}
11255
11256 // This pattern is automatically generated from aarch64_ad.m4
11257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11258 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11259 iRegL src1, iRegL src2,
11260 immI src3) %{
11261 match(Set dst (SubL src1 (URShiftL src2 src3)));
11262
11263 ins_cost(1.9 * INSN_COST);
11264 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11265
11266 ins_encode %{
11267 __ sub(as_Register($dst$$reg),
11268 as_Register($src1$$reg),
11269 as_Register($src2$$reg),
11270 Assembler::LSR,
11271 $src3$$constant & 0x3f);
11272 %}
11273
11274 ins_pipe(ialu_reg_reg_shift);
11275 %}
11276
11277 // This pattern is automatically generated from aarch64_ad.m4
11278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11279 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11280 iRegIorL2I src1, iRegIorL2I src2,
11281 immI src3) %{
11282 match(Set dst (SubI src1 (RShiftI src2 src3)));
11283
11284 ins_cost(1.9 * INSN_COST);
11285 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11286
11287 ins_encode %{
11288 __ subw(as_Register($dst$$reg),
11289 as_Register($src1$$reg),
11290 as_Register($src2$$reg),
11291 Assembler::ASR,
11292 $src3$$constant & 0x1f);
11293 %}
11294
11295 ins_pipe(ialu_reg_reg_shift);
11296 %}
11297
11298 // This pattern is automatically generated from aarch64_ad.m4
11299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11300 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11301 iRegL src1, iRegL src2,
11302 immI src3) %{
11303 match(Set dst (SubL src1 (RShiftL src2 src3)));
11304
11305 ins_cost(1.9 * INSN_COST);
11306 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11307
11308 ins_encode %{
11309 __ sub(as_Register($dst$$reg),
11310 as_Register($src1$$reg),
11311 as_Register($src2$$reg),
11312 Assembler::ASR,
11313 $src3$$constant & 0x3f);
11314 %}
11315
11316 ins_pipe(ialu_reg_reg_shift);
11317 %}
11318
11319 // This pattern is automatically generated from aarch64_ad.m4
11320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11321 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11322 iRegIorL2I src1, iRegIorL2I src2,
11323 immI src3) %{
11324 match(Set dst (SubI src1 (LShiftI src2 src3)));
11325
11326 ins_cost(1.9 * INSN_COST);
11327 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11328
11329 ins_encode %{
11330 __ subw(as_Register($dst$$reg),
11331 as_Register($src1$$reg),
11332 as_Register($src2$$reg),
11333 Assembler::LSL,
11334 $src3$$constant & 0x1f);
11335 %}
11336
11337 ins_pipe(ialu_reg_reg_shift);
11338 %}
11339
11340 // This pattern is automatically generated from aarch64_ad.m4
11341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11342 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11343 iRegL src1, iRegL src2,
11344 immI src3) %{
11345 match(Set dst (SubL src1 (LShiftL src2 src3)));
11346
11347 ins_cost(1.9 * INSN_COST);
11348 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11349
11350 ins_encode %{
11351 __ sub(as_Register($dst$$reg),
11352 as_Register($src1$$reg),
11353 as_Register($src2$$reg),
11354 Assembler::LSL,
11355 $src3$$constant & 0x3f);
11356 %}
11357
11358 ins_pipe(ialu_reg_reg_shift);
11359 %}
11360
11361 // This pattern is automatically generated from aarch64_ad.m4
11362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11363
11364 // Shift Left followed by Shift Right.
11365 // This idiom is used by the compiler for the i2b bytecode etc.
11366 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11367 %{
11368 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11369 ins_cost(INSN_COST * 2);
11370 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11371 ins_encode %{
11372 int lshift = $lshift_count$$constant & 63;
11373 int rshift = $rshift_count$$constant & 63;
11374 int s = 63 - lshift;
11375 int r = (rshift - lshift) & 63;
11376 __ sbfm(as_Register($dst$$reg),
11377 as_Register($src$$reg),
11378 r, s);
11379 %}
11380
11381 ins_pipe(ialu_reg_shift);
11382 %}
11383
11384 // This pattern is automatically generated from aarch64_ad.m4
11385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11386
11387 // Shift Left followed by Shift Right.
11388 // This idiom is used by the compiler for the i2b bytecode etc.
11389 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11390 %{
11391 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11392 ins_cost(INSN_COST * 2);
11393 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11394 ins_encode %{
11395 int lshift = $lshift_count$$constant & 31;
11396 int rshift = $rshift_count$$constant & 31;
11397 int s = 31 - lshift;
11398 int r = (rshift - lshift) & 31;
11399 __ sbfmw(as_Register($dst$$reg),
11400 as_Register($src$$reg),
11401 r, s);
11402 %}
11403
11404 ins_pipe(ialu_reg_shift);
11405 %}
11406
11407 // This pattern is automatically generated from aarch64_ad.m4
11408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11409
11410 // Shift Left followed by Shift Right.
11411 // This idiom is used by the compiler for the i2b bytecode etc.
11412 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11413 %{
11414 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11415 ins_cost(INSN_COST * 2);
11416 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11417 ins_encode %{
11418 int lshift = $lshift_count$$constant & 63;
11419 int rshift = $rshift_count$$constant & 63;
11420 int s = 63 - lshift;
11421 int r = (rshift - lshift) & 63;
11422 __ ubfm(as_Register($dst$$reg),
11423 as_Register($src$$reg),
11424 r, s);
11425 %}
11426
11427 ins_pipe(ialu_reg_shift);
11428 %}
11429
11430 // This pattern is automatically generated from aarch64_ad.m4
11431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11432
11433 // Shift Left followed by Shift Right.
11434 // This idiom is used by the compiler for the i2b bytecode etc.
11435 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11436 %{
11437 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11438 ins_cost(INSN_COST * 2);
11439 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11440 ins_encode %{
11441 int lshift = $lshift_count$$constant & 31;
11442 int rshift = $rshift_count$$constant & 31;
11443 int s = 31 - lshift;
11444 int r = (rshift - lshift) & 31;
11445 __ ubfmw(as_Register($dst$$reg),
11446 as_Register($src$$reg),
11447 r, s);
11448 %}
11449
11450 ins_pipe(ialu_reg_shift);
11451 %}
11452
11453 // Bitfield extract with shift & mask
11454
11455 // This pattern is automatically generated from aarch64_ad.m4
11456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11457 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11458 %{
11459 match(Set dst (AndI (URShiftI src rshift) mask));
11460 // Make sure we are not going to exceed what ubfxw can do.
11461 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11462
11463 ins_cost(INSN_COST);
11464 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11465 ins_encode %{
11466 int rshift = $rshift$$constant & 31;
11467 intptr_t mask = $mask$$constant;
11468 int width = exact_log2(mask+1);
11469 __ ubfxw(as_Register($dst$$reg),
11470 as_Register($src$$reg), rshift, width);
11471 %}
11472 ins_pipe(ialu_reg_shift);
11473 %}
11474
11475 // This pattern is automatically generated from aarch64_ad.m4
11476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11477 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11478 %{
11479 match(Set dst (AndL (URShiftL src rshift) mask));
11480 // Make sure we are not going to exceed what ubfx can do.
11481 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11482
11483 ins_cost(INSN_COST);
11484 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11485 ins_encode %{
11486 int rshift = $rshift$$constant & 63;
11487 intptr_t mask = $mask$$constant;
11488 int width = exact_log2_long(mask+1);
11489 __ ubfx(as_Register($dst$$reg),
11490 as_Register($src$$reg), rshift, width);
11491 %}
11492 ins_pipe(ialu_reg_shift);
11493 %}
11494
11495
11496 // This pattern is automatically generated from aarch64_ad.m4
11497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11498
11499 // We can use ubfx when extending an And with a mask when we know mask
11500 // is positive. We know that because immI_bitmask guarantees it.
11501 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11502 %{
11503 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11504 // Make sure we are not going to exceed what ubfxw can do.
11505 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11506
11507 ins_cost(INSN_COST * 2);
11508 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11509 ins_encode %{
11510 int rshift = $rshift$$constant & 31;
11511 intptr_t mask = $mask$$constant;
11512 int width = exact_log2(mask+1);
11513 __ ubfx(as_Register($dst$$reg),
11514 as_Register($src$$reg), rshift, width);
11515 %}
11516 ins_pipe(ialu_reg_shift);
11517 %}
11518
11519
11520 // This pattern is automatically generated from aarch64_ad.m4
11521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11522
11523 // We can use ubfiz when masking by a positive number and then left shifting the result.
11524 // We know that the mask is positive because immI_bitmask guarantees it.
11525 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11526 %{
11527 match(Set dst (LShiftI (AndI src mask) lshift));
11528 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11529
11530 ins_cost(INSN_COST);
11531 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11532 ins_encode %{
11533 int lshift = $lshift$$constant & 31;
11534 intptr_t mask = $mask$$constant;
11535 int width = exact_log2(mask+1);
11536 __ ubfizw(as_Register($dst$$reg),
11537 as_Register($src$$reg), lshift, width);
11538 %}
11539 ins_pipe(ialu_reg_shift);
11540 %}
11541
11542 // This pattern is automatically generated from aarch64_ad.m4
11543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11544
11545 // We can use ubfiz when masking by a positive number and then left shifting the result.
11546 // We know that the mask is positive because immL_bitmask guarantees it.
11547 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11548 %{
11549 match(Set dst (LShiftL (AndL src mask) lshift));
11550 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11551
11552 ins_cost(INSN_COST);
11553 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11554 ins_encode %{
11555 int lshift = $lshift$$constant & 63;
11556 intptr_t mask = $mask$$constant;
11557 int width = exact_log2_long(mask+1);
11558 __ ubfiz(as_Register($dst$$reg),
11559 as_Register($src$$reg), lshift, width);
11560 %}
11561 ins_pipe(ialu_reg_shift);
11562 %}
11563
11564 // This pattern is automatically generated from aarch64_ad.m4
11565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11566
11567 // We can use ubfiz when masking by a positive number and then left shifting the result.
11568 // We know that the mask is positive because immI_bitmask guarantees it.
11569 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11570 %{
11571 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11572 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11573
11574 ins_cost(INSN_COST);
11575 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11576 ins_encode %{
11577 int lshift = $lshift$$constant & 31;
11578 intptr_t mask = $mask$$constant;
11579 int width = exact_log2(mask+1);
11580 __ ubfizw(as_Register($dst$$reg),
11581 as_Register($src$$reg), lshift, width);
11582 %}
11583 ins_pipe(ialu_reg_shift);
11584 %}
11585
11586 // This pattern is automatically generated from aarch64_ad.m4
11587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11588
11589 // We can use ubfiz when masking by a positive number and then left shifting the result.
11590 // We know that the mask is positive because immL_bitmask guarantees it.
11591 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11592 %{
11593 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11594 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11595
11596 ins_cost(INSN_COST);
11597 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11598 ins_encode %{
11599 int lshift = $lshift$$constant & 63;
11600 intptr_t mask = $mask$$constant;
11601 int width = exact_log2_long(mask+1);
11602 __ ubfiz(as_Register($dst$$reg),
11603 as_Register($src$$reg), lshift, width);
11604 %}
11605 ins_pipe(ialu_reg_shift);
11606 %}
11607
11608
11609 // This pattern is automatically generated from aarch64_ad.m4
11610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11611
11612 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11613 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11614 %{
11615 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11616 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11617
11618 ins_cost(INSN_COST);
11619 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11620 ins_encode %{
11621 int lshift = $lshift$$constant & 63;
11622 intptr_t mask = $mask$$constant;
11623 int width = exact_log2(mask+1);
11624 __ ubfiz(as_Register($dst$$reg),
11625 as_Register($src$$reg), lshift, width);
11626 %}
11627 ins_pipe(ialu_reg_shift);
11628 %}
11629
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632
11633 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11634 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11635 %{
11636 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11637 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11638
11639 ins_cost(INSN_COST);
11640 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11641 ins_encode %{
11642 int lshift = $lshift$$constant & 31;
11643 intptr_t mask = $mask$$constant;
11644 int width = exact_log2(mask+1);
11645 __ ubfiz(as_Register($dst$$reg),
11646 as_Register($src$$reg), lshift, width);
11647 %}
11648 ins_pipe(ialu_reg_shift);
11649 %}
11650
11651 // This pattern is automatically generated from aarch64_ad.m4
11652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11653
11654 // Can skip int2long conversions after AND with small bitmask
11655 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11656 %{
11657 match(Set dst (ConvI2L (AndI src msk)));
11658 ins_cost(INSN_COST);
11659 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11660 ins_encode %{
11661 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11662 %}
11663 ins_pipe(ialu_reg_shift);
11664 %}
11665
11666
11667 // Rotations
11668
11669 // This pattern is automatically generated from aarch64_ad.m4
11670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11671 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11672 %{
11673 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11674 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11675
11676 ins_cost(INSN_COST);
11677 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11678
11679 ins_encode %{
11680 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11681 $rshift$$constant & 63);
11682 %}
11683 ins_pipe(ialu_reg_reg_extr);
11684 %}
11685
11686
11687 // This pattern is automatically generated from aarch64_ad.m4
11688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11689 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11690 %{
11691 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11692 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11693
11694 ins_cost(INSN_COST);
11695 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11696
11697 ins_encode %{
11698 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11699 $rshift$$constant & 31);
11700 %}
11701 ins_pipe(ialu_reg_reg_extr);
11702 %}
11703
11704
11705 // This pattern is automatically generated from aarch64_ad.m4
11706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11707 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11708 %{
11709 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11710 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11711
11712 ins_cost(INSN_COST);
11713 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11714
11715 ins_encode %{
11716 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11717 $rshift$$constant & 63);
11718 %}
11719 ins_pipe(ialu_reg_reg_extr);
11720 %}
11721
11722
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11726 %{
11727 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11728 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11729
11730 ins_cost(INSN_COST);
11731 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11732
11733 ins_encode %{
11734 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11735 $rshift$$constant & 31);
11736 %}
11737 ins_pipe(ialu_reg_reg_extr);
11738 %}
11739
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11743 %{
11744 match(Set dst (RotateRight src shift));
11745
11746 ins_cost(INSN_COST);
11747 format %{ "ror $dst, $src, $shift" %}
11748
11749 ins_encode %{
11750 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11751 $shift$$constant & 0x1f);
11752 %}
11753 ins_pipe(ialu_reg_reg_vshift);
11754 %}
11755
11756 // This pattern is automatically generated from aarch64_ad.m4
11757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11758 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11759 %{
11760 match(Set dst (RotateRight src shift));
11761
11762 ins_cost(INSN_COST);
11763 format %{ "ror $dst, $src, $shift" %}
11764
11765 ins_encode %{
11766 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11767 $shift$$constant & 0x3f);
11768 %}
11769 ins_pipe(ialu_reg_reg_vshift);
11770 %}
11771
11772 // This pattern is automatically generated from aarch64_ad.m4
11773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11774 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11775 %{
11776 match(Set dst (RotateRight src shift));
11777
11778 ins_cost(INSN_COST);
11779 format %{ "ror $dst, $src, $shift" %}
11780
11781 ins_encode %{
11782 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11783 %}
11784 ins_pipe(ialu_reg_reg_vshift);
11785 %}
11786
11787 // This pattern is automatically generated from aarch64_ad.m4
11788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11789 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11790 %{
11791 match(Set dst (RotateRight src shift));
11792
11793 ins_cost(INSN_COST);
11794 format %{ "ror $dst, $src, $shift" %}
11795
11796 ins_encode %{
11797 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11798 %}
11799 ins_pipe(ialu_reg_reg_vshift);
11800 %}
11801
11802 // This pattern is automatically generated from aarch64_ad.m4
11803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11804 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11805 %{
11806 match(Set dst (RotateLeft src shift));
11807
11808 ins_cost(INSN_COST);
11809 format %{ "rol $dst, $src, $shift" %}
11810
11811 ins_encode %{
11812 __ subw(rscratch1, zr, as_Register($shift$$reg));
11813 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11814 %}
11815 ins_pipe(ialu_reg_reg_vshift);
11816 %}
11817
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11821 %{
11822 match(Set dst (RotateLeft src shift));
11823
11824 ins_cost(INSN_COST);
11825 format %{ "rol $dst, $src, $shift" %}
11826
11827 ins_encode %{
11828 __ subw(rscratch1, zr, as_Register($shift$$reg));
11829 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11830 %}
11831 ins_pipe(ialu_reg_reg_vshift);
11832 %}
11833
11834
11835 // Add/subtract (extended)
11836
11837 // This pattern is automatically generated from aarch64_ad.m4
11838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11839 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11840 %{
11841 match(Set dst (AddL src1 (ConvI2L src2)));
11842 ins_cost(INSN_COST);
11843 format %{ "add $dst, $src1, $src2, sxtw" %}
11844
11845 ins_encode %{
11846 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11847 as_Register($src2$$reg), ext::sxtw);
11848 %}
11849 ins_pipe(ialu_reg_reg);
11850 %}
11851
11852 // This pattern is automatically generated from aarch64_ad.m4
11853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11854 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11855 %{
11856 match(Set dst (SubL src1 (ConvI2L src2)));
11857 ins_cost(INSN_COST);
11858 format %{ "sub $dst, $src1, $src2, sxtw" %}
11859
11860 ins_encode %{
11861 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11862 as_Register($src2$$reg), ext::sxtw);
11863 %}
11864 ins_pipe(ialu_reg_reg);
11865 %}
11866
11867 // This pattern is automatically generated from aarch64_ad.m4
11868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11869 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11870 %{
11871 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11872 ins_cost(INSN_COST);
11873 format %{ "add $dst, $src1, $src2, sxth" %}
11874
11875 ins_encode %{
11876 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11877 as_Register($src2$$reg), ext::sxth);
11878 %}
11879 ins_pipe(ialu_reg_reg);
11880 %}
11881
11882 // This pattern is automatically generated from aarch64_ad.m4
11883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11884 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11885 %{
11886 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11887 ins_cost(INSN_COST);
11888 format %{ "add $dst, $src1, $src2, sxtb" %}
11889
11890 ins_encode %{
11891 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11892 as_Register($src2$$reg), ext::sxtb);
11893 %}
11894 ins_pipe(ialu_reg_reg);
11895 %}
11896
11897 // This pattern is automatically generated from aarch64_ad.m4
11898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11899 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11900 %{
11901 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11902 ins_cost(INSN_COST);
11903 format %{ "add $dst, $src1, $src2, uxtb" %}
11904
11905 ins_encode %{
11906 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11907 as_Register($src2$$reg), ext::uxtb);
11908 %}
11909 ins_pipe(ialu_reg_reg);
11910 %}
11911
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11915 %{
11916 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11917 ins_cost(INSN_COST);
11918 format %{ "add $dst, $src1, $src2, sxth" %}
11919
11920 ins_encode %{
11921 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11922 as_Register($src2$$reg), ext::sxth);
11923 %}
11924 ins_pipe(ialu_reg_reg);
11925 %}
11926
11927 // This pattern is automatically generated from aarch64_ad.m4
11928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11929 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11930 %{
11931 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11932 ins_cost(INSN_COST);
11933 format %{ "add $dst, $src1, $src2, sxtw" %}
11934
11935 ins_encode %{
11936 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11937 as_Register($src2$$reg), ext::sxtw);
11938 %}
11939 ins_pipe(ialu_reg_reg);
11940 %}
11941
11942 // This pattern is automatically generated from aarch64_ad.m4
11943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11944 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11945 %{
11946 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11947 ins_cost(INSN_COST);
11948 format %{ "add $dst, $src1, $src2, sxtb" %}
11949
11950 ins_encode %{
11951 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11952 as_Register($src2$$reg), ext::sxtb);
11953 %}
11954 ins_pipe(ialu_reg_reg);
11955 %}
11956
11957 // This pattern is automatically generated from aarch64_ad.m4
11958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11959 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11960 %{
11961 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11962 ins_cost(INSN_COST);
11963 format %{ "add $dst, $src1, $src2, uxtb" %}
11964
11965 ins_encode %{
11966 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11967 as_Register($src2$$reg), ext::uxtb);
11968 %}
11969 ins_pipe(ialu_reg_reg);
11970 %}
11971
11972 // This pattern is automatically generated from aarch64_ad.m4
11973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11974 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11975 %{
11976 match(Set dst (AddI src1 (AndI src2 mask)));
11977 ins_cost(INSN_COST);
11978 format %{ "addw $dst, $src1, $src2, uxtb" %}
11979
11980 ins_encode %{
11981 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11982 as_Register($src2$$reg), ext::uxtb);
11983 %}
11984 ins_pipe(ialu_reg_reg);
11985 %}
11986
11987 // This pattern is automatically generated from aarch64_ad.m4
11988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11989 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11990 %{
11991 match(Set dst (AddI src1 (AndI src2 mask)));
11992 ins_cost(INSN_COST);
11993 format %{ "addw $dst, $src1, $src2, uxth" %}
11994
11995 ins_encode %{
11996 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11997 as_Register($src2$$reg), ext::uxth);
11998 %}
11999 ins_pipe(ialu_reg_reg);
12000 %}
12001
12002 // This pattern is automatically generated from aarch64_ad.m4
12003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12004 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12005 %{
12006 match(Set dst (AddL src1 (AndL src2 mask)));
12007 ins_cost(INSN_COST);
12008 format %{ "add $dst, $src1, $src2, uxtb" %}
12009
12010 ins_encode %{
12011 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12012 as_Register($src2$$reg), ext::uxtb);
12013 %}
12014 ins_pipe(ialu_reg_reg);
12015 %}
12016
12017 // This pattern is automatically generated from aarch64_ad.m4
12018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12019 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12020 %{
12021 match(Set dst (AddL src1 (AndL src2 mask)));
12022 ins_cost(INSN_COST);
12023 format %{ "add $dst, $src1, $src2, uxth" %}
12024
12025 ins_encode %{
12026 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12027 as_Register($src2$$reg), ext::uxth);
12028 %}
12029 ins_pipe(ialu_reg_reg);
12030 %}
12031
12032 // This pattern is automatically generated from aarch64_ad.m4
12033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12034 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12035 %{
12036 match(Set dst (AddL src1 (AndL src2 mask)));
12037 ins_cost(INSN_COST);
12038 format %{ "add $dst, $src1, $src2, uxtw" %}
12039
12040 ins_encode %{
12041 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12042 as_Register($src2$$reg), ext::uxtw);
12043 %}
12044 ins_pipe(ialu_reg_reg);
12045 %}
12046
12047 // This pattern is automatically generated from aarch64_ad.m4
12048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12049 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12050 %{
12051 match(Set dst (SubI src1 (AndI src2 mask)));
12052 ins_cost(INSN_COST);
12053 format %{ "subw $dst, $src1, $src2, uxtb" %}
12054
12055 ins_encode %{
12056 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12057 as_Register($src2$$reg), ext::uxtb);
12058 %}
12059 ins_pipe(ialu_reg_reg);
12060 %}
12061
12062 // This pattern is automatically generated from aarch64_ad.m4
12063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12064 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12065 %{
12066 match(Set dst (SubI src1 (AndI src2 mask)));
12067 ins_cost(INSN_COST);
12068 format %{ "subw $dst, $src1, $src2, uxth" %}
12069
12070 ins_encode %{
12071 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12072 as_Register($src2$$reg), ext::uxth);
12073 %}
12074 ins_pipe(ialu_reg_reg);
12075 %}
12076
12077 // This pattern is automatically generated from aarch64_ad.m4
12078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12079 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12080 %{
12081 match(Set dst (SubL src1 (AndL src2 mask)));
12082 ins_cost(INSN_COST);
12083 format %{ "sub $dst, $src1, $src2, uxtb" %}
12084
12085 ins_encode %{
12086 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12087 as_Register($src2$$reg), ext::uxtb);
12088 %}
12089 ins_pipe(ialu_reg_reg);
12090 %}
12091
12092 // This pattern is automatically generated from aarch64_ad.m4
12093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12094 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12095 %{
12096 match(Set dst (SubL src1 (AndL src2 mask)));
12097 ins_cost(INSN_COST);
12098 format %{ "sub $dst, $src1, $src2, uxth" %}
12099
12100 ins_encode %{
12101 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12102 as_Register($src2$$reg), ext::uxth);
12103 %}
12104 ins_pipe(ialu_reg_reg);
12105 %}
12106
12107 // This pattern is automatically generated from aarch64_ad.m4
12108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12109 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12110 %{
12111 match(Set dst (SubL src1 (AndL src2 mask)));
12112 ins_cost(INSN_COST);
12113 format %{ "sub $dst, $src1, $src2, uxtw" %}
12114
12115 ins_encode %{
12116 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12117 as_Register($src2$$reg), ext::uxtw);
12118 %}
12119 ins_pipe(ialu_reg_reg);
12120 %}
12121
12122
12123 // This pattern is automatically generated from aarch64_ad.m4
12124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12125 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12126 %{
12127 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12128 ins_cost(1.9 * INSN_COST);
12129 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12130
12131 ins_encode %{
12132 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12133 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12134 %}
12135 ins_pipe(ialu_reg_reg_shift);
12136 %}
12137
12138 // This pattern is automatically generated from aarch64_ad.m4
12139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12140 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12141 %{
12142 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12143 ins_cost(1.9 * INSN_COST);
12144 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12145
12146 ins_encode %{
12147 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12148 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12149 %}
12150 ins_pipe(ialu_reg_reg_shift);
12151 %}
12152
12153 // This pattern is automatically generated from aarch64_ad.m4
12154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12155 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12156 %{
12157 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12158 ins_cost(1.9 * INSN_COST);
12159 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12160
12161 ins_encode %{
12162 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12163 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12164 %}
12165 ins_pipe(ialu_reg_reg_shift);
12166 %}
12167
12168 // This pattern is automatically generated from aarch64_ad.m4
12169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12170 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12171 %{
12172 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12173 ins_cost(1.9 * INSN_COST);
12174 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12175
12176 ins_encode %{
12177 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12178 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12179 %}
12180 ins_pipe(ialu_reg_reg_shift);
12181 %}
12182
12183 // This pattern is automatically generated from aarch64_ad.m4
12184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12185 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12186 %{
12187 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12188 ins_cost(1.9 * INSN_COST);
12189 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12190
12191 ins_encode %{
12192 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12193 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12194 %}
12195 ins_pipe(ialu_reg_reg_shift);
12196 %}
12197
12198 // This pattern is automatically generated from aarch64_ad.m4
12199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12200 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12201 %{
12202 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12203 ins_cost(1.9 * INSN_COST);
12204 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12205
12206 ins_encode %{
12207 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12208 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12209 %}
12210 ins_pipe(ialu_reg_reg_shift);
12211 %}
12212
12213 // This pattern is automatically generated from aarch64_ad.m4
12214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12215 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12216 %{
12217 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12218 ins_cost(1.9 * INSN_COST);
12219 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12220
12221 ins_encode %{
12222 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12223 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12224 %}
12225 ins_pipe(ialu_reg_reg_shift);
12226 %}
12227
12228 // This pattern is automatically generated from aarch64_ad.m4
12229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12230 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12231 %{
12232 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12233 ins_cost(1.9 * INSN_COST);
12234 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12235
12236 ins_encode %{
12237 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12238 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12239 %}
12240 ins_pipe(ialu_reg_reg_shift);
12241 %}
12242
12243 // This pattern is automatically generated from aarch64_ad.m4
12244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12245 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12246 %{
12247 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12248 ins_cost(1.9 * INSN_COST);
12249 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12250
12251 ins_encode %{
12252 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12253 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12254 %}
12255 ins_pipe(ialu_reg_reg_shift);
12256 %}
12257
12258 // This pattern is automatically generated from aarch64_ad.m4
12259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12260 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12261 %{
12262 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12263 ins_cost(1.9 * INSN_COST);
12264 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12265
12266 ins_encode %{
12267 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12268 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12269 %}
12270 ins_pipe(ialu_reg_reg_shift);
12271 %}
12272
12273 // This pattern is automatically generated from aarch64_ad.m4
12274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12275 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12276 %{
12277 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12278 ins_cost(1.9 * INSN_COST);
12279 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12280
12281 ins_encode %{
12282 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12283 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12284 %}
12285 ins_pipe(ialu_reg_reg_shift);
12286 %}
12287
12288 // This pattern is automatically generated from aarch64_ad.m4
12289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12290 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12291 %{
12292 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12293 ins_cost(1.9 * INSN_COST);
12294 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12295
12296 ins_encode %{
12297 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12298 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12299 %}
12300 ins_pipe(ialu_reg_reg_shift);
12301 %}
12302
12303 // This pattern is automatically generated from aarch64_ad.m4
12304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12305 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12306 %{
12307 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12308 ins_cost(1.9 * INSN_COST);
12309 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12310
12311 ins_encode %{
12312 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12313 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12314 %}
12315 ins_pipe(ialu_reg_reg_shift);
12316 %}
12317
12318 // This pattern is automatically generated from aarch64_ad.m4
12319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12320 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12321 %{
12322 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12323 ins_cost(1.9 * INSN_COST);
12324 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12325
12326 ins_encode %{
12327 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12328 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12329 %}
12330 ins_pipe(ialu_reg_reg_shift);
12331 %}
12332
12333 // This pattern is automatically generated from aarch64_ad.m4
12334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12335 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12336 %{
12337 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12338 ins_cost(1.9 * INSN_COST);
12339 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12340
12341 ins_encode %{
12342 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12343 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12344 %}
12345 ins_pipe(ialu_reg_reg_shift);
12346 %}
12347
12348 // This pattern is automatically generated from aarch64_ad.m4
12349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12350 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12351 %{
12352 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12353 ins_cost(1.9 * INSN_COST);
12354 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12355
12356 ins_encode %{
12357 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12358 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12359 %}
12360 ins_pipe(ialu_reg_reg_shift);
12361 %}
12362
12363 // This pattern is automatically generated from aarch64_ad.m4
12364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12365 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12366 %{
12367 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12368 ins_cost(1.9 * INSN_COST);
12369 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12370
12371 ins_encode %{
12372 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12373 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12374 %}
12375 ins_pipe(ialu_reg_reg_shift);
12376 %}
12377
12378 // This pattern is automatically generated from aarch64_ad.m4
12379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12380 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12381 %{
12382 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12383 ins_cost(1.9 * INSN_COST);
12384 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12385
12386 ins_encode %{
12387 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12388 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12389 %}
12390 ins_pipe(ialu_reg_reg_shift);
12391 %}
12392
12393 // This pattern is automatically generated from aarch64_ad.m4
12394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12395 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12396 %{
12397 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12398 ins_cost(1.9 * INSN_COST);
12399 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12400
12401 ins_encode %{
12402 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12403 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12404 %}
12405 ins_pipe(ialu_reg_reg_shift);
12406 %}
12407
12408 // This pattern is automatically generated from aarch64_ad.m4
12409 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12410 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12411 %{
12412 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12413 ins_cost(1.9 * INSN_COST);
12414 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12415
12416 ins_encode %{
12417 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12418 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12419 %}
12420 ins_pipe(ialu_reg_reg_shift);
12421 %}
12422
12423 // This pattern is automatically generated from aarch64_ad.m4
12424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12425 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12426 %{
12427 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12428 ins_cost(1.9 * INSN_COST);
12429 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12430
12431 ins_encode %{
12432 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12433 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12434 %}
12435 ins_pipe(ialu_reg_reg_shift);
12436 %}
12437
12438 // This pattern is automatically generated from aarch64_ad.m4
12439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12440 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12441 %{
12442 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12443 ins_cost(1.9 * INSN_COST);
12444 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12445
12446 ins_encode %{
12447 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12448 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12449 %}
12450 ins_pipe(ialu_reg_reg_shift);
12451 %}
12452
12453 // This pattern is automatically generated from aarch64_ad.m4
12454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12455 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12456 %{
12457 effect(DEF dst, USE src1, USE src2, USE cr);
12458 ins_cost(INSN_COST * 2);
12459 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12460
12461 ins_encode %{
12462 __ cselw($dst$$Register,
12463 $src1$$Register,
12464 $src2$$Register,
12465 Assembler::LT);
12466 %}
12467 ins_pipe(icond_reg_reg);
12468 %}
12469
12470 // This pattern is automatically generated from aarch64_ad.m4
12471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12472 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12473 %{
12474 effect(DEF dst, USE src1, USE src2, USE cr);
12475 ins_cost(INSN_COST * 2);
12476 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12477
12478 ins_encode %{
12479 __ cselw($dst$$Register,
12480 $src1$$Register,
12481 $src2$$Register,
12482 Assembler::GT);
12483 %}
12484 ins_pipe(icond_reg_reg);
12485 %}
12486
12487 // This pattern is automatically generated from aarch64_ad.m4
12488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12489 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12490 %{
12491 effect(DEF dst, USE src1, USE cr);
12492 ins_cost(INSN_COST * 2);
12493 format %{ "cselw $dst, $src1, zr lt\t" %}
12494
12495 ins_encode %{
12496 __ cselw($dst$$Register,
12497 $src1$$Register,
12498 zr,
12499 Assembler::LT);
12500 %}
12501 ins_pipe(icond_reg);
12502 %}
12503
12504 // This pattern is automatically generated from aarch64_ad.m4
12505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12506 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12507 %{
12508 effect(DEF dst, USE src1, USE cr);
12509 ins_cost(INSN_COST * 2);
12510 format %{ "cselw $dst, $src1, zr gt\t" %}
12511
12512 ins_encode %{
12513 __ cselw($dst$$Register,
12514 $src1$$Register,
12515 zr,
12516 Assembler::GT);
12517 %}
12518 ins_pipe(icond_reg);
12519 %}
12520
12521 // This pattern is automatically generated from aarch64_ad.m4
12522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12523 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12524 %{
12525 effect(DEF dst, USE src1, USE cr);
12526 ins_cost(INSN_COST * 2);
12527 format %{ "csincw $dst, $src1, zr le\t" %}
12528
12529 ins_encode %{
12530 __ csincw($dst$$Register,
12531 $src1$$Register,
12532 zr,
12533 Assembler::LE);
12534 %}
12535 ins_pipe(icond_reg);
12536 %}
12537
12538 // This pattern is automatically generated from aarch64_ad.m4
12539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12540 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12541 %{
12542 effect(DEF dst, USE src1, USE cr);
12543 ins_cost(INSN_COST * 2);
12544 format %{ "csincw $dst, $src1, zr gt\t" %}
12545
12546 ins_encode %{
12547 __ csincw($dst$$Register,
12548 $src1$$Register,
12549 zr,
12550 Assembler::GT);
12551 %}
12552 ins_pipe(icond_reg);
12553 %}
12554
12555 // This pattern is automatically generated from aarch64_ad.m4
12556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12557 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12558 %{
12559 effect(DEF dst, USE src1, USE cr);
12560 ins_cost(INSN_COST * 2);
12561 format %{ "csinvw $dst, $src1, zr lt\t" %}
12562
12563 ins_encode %{
12564 __ csinvw($dst$$Register,
12565 $src1$$Register,
12566 zr,
12567 Assembler::LT);
12568 %}
12569 ins_pipe(icond_reg);
12570 %}
12571
12572 // This pattern is automatically generated from aarch64_ad.m4
12573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12574 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12575 %{
12576 effect(DEF dst, USE src1, USE cr);
12577 ins_cost(INSN_COST * 2);
12578 format %{ "csinvw $dst, $src1, zr ge\t" %}
12579
12580 ins_encode %{
12581 __ csinvw($dst$$Register,
12582 $src1$$Register,
12583 zr,
12584 Assembler::GE);
12585 %}
12586 ins_pipe(icond_reg);
12587 %}
12588
12589 // This pattern is automatically generated from aarch64_ad.m4
12590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12591 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12592 %{
12593 match(Set dst (MinI src imm));
12594 ins_cost(INSN_COST * 3);
12595 expand %{
12596 rFlagsReg cr;
12597 compI_reg_imm0(cr, src);
12598 cmovI_reg_imm0_lt(dst, src, cr);
12599 %}
12600 %}
12601
12602 // This pattern is automatically generated from aarch64_ad.m4
12603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12604 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12605 %{
12606 match(Set dst (MinI imm src));
12607 ins_cost(INSN_COST * 3);
12608 expand %{
12609 rFlagsReg cr;
12610 compI_reg_imm0(cr, src);
12611 cmovI_reg_imm0_lt(dst, src, cr);
12612 %}
12613 %}
12614
12615 // This pattern is automatically generated from aarch64_ad.m4
12616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12617 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12618 %{
12619 match(Set dst (MinI src imm));
12620 ins_cost(INSN_COST * 3);
12621 expand %{
12622 rFlagsReg cr;
12623 compI_reg_imm0(cr, src);
12624 cmovI_reg_imm1_le(dst, src, cr);
12625 %}
12626 %}
12627
12628 // This pattern is automatically generated from aarch64_ad.m4
12629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12630 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12631 %{
12632 match(Set dst (MinI imm src));
12633 ins_cost(INSN_COST * 3);
12634 expand %{
12635 rFlagsReg cr;
12636 compI_reg_imm0(cr, src);
12637 cmovI_reg_imm1_le(dst, src, cr);
12638 %}
12639 %}
12640
12641 // This pattern is automatically generated from aarch64_ad.m4
12642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12643 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12644 %{
12645 match(Set dst (MinI src imm));
12646 ins_cost(INSN_COST * 3);
12647 expand %{
12648 rFlagsReg cr;
12649 compI_reg_imm0(cr, src);
12650 cmovI_reg_immM1_lt(dst, src, cr);
12651 %}
12652 %}
12653
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12657 %{
12658 match(Set dst (MinI imm src));
12659 ins_cost(INSN_COST * 3);
12660 expand %{
12661 rFlagsReg cr;
12662 compI_reg_imm0(cr, src);
12663 cmovI_reg_immM1_lt(dst, src, cr);
12664 %}
12665 %}
12666
12667 // This pattern is automatically generated from aarch64_ad.m4
12668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12669 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12670 %{
12671 match(Set dst (MaxI src imm));
12672 ins_cost(INSN_COST * 3);
12673 expand %{
12674 rFlagsReg cr;
12675 compI_reg_imm0(cr, src);
12676 cmovI_reg_imm0_gt(dst, src, cr);
12677 %}
12678 %}
12679
12680 // This pattern is automatically generated from aarch64_ad.m4
12681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12682 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12683 %{
12684 match(Set dst (MaxI imm src));
12685 ins_cost(INSN_COST * 3);
12686 expand %{
12687 rFlagsReg cr;
12688 compI_reg_imm0(cr, src);
12689 cmovI_reg_imm0_gt(dst, src, cr);
12690 %}
12691 %}
12692
12693 // This pattern is automatically generated from aarch64_ad.m4
12694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12695 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12696 %{
12697 match(Set dst (MaxI src imm));
12698 ins_cost(INSN_COST * 3);
12699 expand %{
12700 rFlagsReg cr;
12701 compI_reg_imm0(cr, src);
12702 cmovI_reg_imm1_gt(dst, src, cr);
12703 %}
12704 %}
12705
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12709 %{
12710 match(Set dst (MaxI imm src));
12711 ins_cost(INSN_COST * 3);
12712 expand %{
12713 rFlagsReg cr;
12714 compI_reg_imm0(cr, src);
12715 cmovI_reg_imm1_gt(dst, src, cr);
12716 %}
12717 %}
12718
12719 // This pattern is automatically generated from aarch64_ad.m4
12720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12721 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12722 %{
12723 match(Set dst (MaxI src imm));
12724 ins_cost(INSN_COST * 3);
12725 expand %{
12726 rFlagsReg cr;
12727 compI_reg_imm0(cr, src);
12728 cmovI_reg_immM1_ge(dst, src, cr);
12729 %}
12730 %}
12731
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12735 %{
12736 match(Set dst (MaxI imm src));
12737 ins_cost(INSN_COST * 3);
12738 expand %{
12739 rFlagsReg cr;
12740 compI_reg_imm0(cr, src);
12741 cmovI_reg_immM1_ge(dst, src, cr);
12742 %}
12743 %}
12744
12745 // This pattern is automatically generated from aarch64_ad.m4
12746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12747 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12748 %{
12749 match(Set dst (ReverseI src));
12750 ins_cost(INSN_COST);
12751 format %{ "rbitw $dst, $src" %}
12752 ins_encode %{
12753 __ rbitw($dst$$Register, $src$$Register);
12754 %}
12755 ins_pipe(ialu_reg);
12756 %}
12757
12758 // This pattern is automatically generated from aarch64_ad.m4
12759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12760 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12761 %{
12762 match(Set dst (ReverseL src));
12763 ins_cost(INSN_COST);
12764 format %{ "rbit $dst, $src" %}
12765 ins_encode %{
12766 __ rbit($dst$$Register, $src$$Register);
12767 %}
12768 ins_pipe(ialu_reg);
12769 %}
12770
12771
12772 // END This section of the file is automatically generated. Do not edit --------------
12773
12774
12775 // ============================================================================
12776 // Floating Point Arithmetic Instructions
12777
12778 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12779 match(Set dst (AddHF src1 src2));
12780 format %{ "faddh $dst, $src1, $src2" %}
12781 ins_encode %{
12782 __ faddh($dst$$FloatRegister,
12783 $src1$$FloatRegister,
12784 $src2$$FloatRegister);
12785 %}
12786 ins_pipe(fp_dop_reg_reg_s);
12787 %}
12788
12789 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12790 match(Set dst (AddF src1 src2));
12791
12792 ins_cost(INSN_COST * 5);
12793 format %{ "fadds $dst, $src1, $src2" %}
12794
12795 ins_encode %{
12796 __ fadds(as_FloatRegister($dst$$reg),
12797 as_FloatRegister($src1$$reg),
12798 as_FloatRegister($src2$$reg));
12799 %}
12800
12801 ins_pipe(fp_dop_reg_reg_s);
12802 %}
12803
12804 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12805 match(Set dst (AddD src1 src2));
12806
12807 ins_cost(INSN_COST * 5);
12808 format %{ "faddd $dst, $src1, $src2" %}
12809
12810 ins_encode %{
12811 __ faddd(as_FloatRegister($dst$$reg),
12812 as_FloatRegister($src1$$reg),
12813 as_FloatRegister($src2$$reg));
12814 %}
12815
12816 ins_pipe(fp_dop_reg_reg_d);
12817 %}
12818
12819 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12820 match(Set dst (SubHF src1 src2));
12821 format %{ "fsubh $dst, $src1, $src2" %}
12822 ins_encode %{
12823 __ fsubh($dst$$FloatRegister,
12824 $src1$$FloatRegister,
12825 $src2$$FloatRegister);
12826 %}
12827 ins_pipe(fp_dop_reg_reg_s);
12828 %}
12829
12830 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12831 match(Set dst (SubF src1 src2));
12832
12833 ins_cost(INSN_COST * 5);
12834 format %{ "fsubs $dst, $src1, $src2" %}
12835
12836 ins_encode %{
12837 __ fsubs(as_FloatRegister($dst$$reg),
12838 as_FloatRegister($src1$$reg),
12839 as_FloatRegister($src2$$reg));
12840 %}
12841
12842 ins_pipe(fp_dop_reg_reg_s);
12843 %}
12844
12845 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12846 match(Set dst (SubD src1 src2));
12847
12848 ins_cost(INSN_COST * 5);
12849 format %{ "fsubd $dst, $src1, $src2" %}
12850
12851 ins_encode %{
12852 __ fsubd(as_FloatRegister($dst$$reg),
12853 as_FloatRegister($src1$$reg),
12854 as_FloatRegister($src2$$reg));
12855 %}
12856
12857 ins_pipe(fp_dop_reg_reg_d);
12858 %}
12859
12860 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12861 match(Set dst (MulHF src1 src2));
12862 format %{ "fmulh $dst, $src1, $src2" %}
12863 ins_encode %{
12864 __ fmulh($dst$$FloatRegister,
12865 $src1$$FloatRegister,
12866 $src2$$FloatRegister);
12867 %}
12868 ins_pipe(fp_dop_reg_reg_s);
12869 %}
12870
12871 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12872 match(Set dst (MulF src1 src2));
12873
12874 ins_cost(INSN_COST * 6);
12875 format %{ "fmuls $dst, $src1, $src2" %}
12876
12877 ins_encode %{
12878 __ fmuls(as_FloatRegister($dst$$reg),
12879 as_FloatRegister($src1$$reg),
12880 as_FloatRegister($src2$$reg));
12881 %}
12882
12883 ins_pipe(fp_dop_reg_reg_s);
12884 %}
12885
12886 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12887 match(Set dst (MulD src1 src2));
12888
12889 ins_cost(INSN_COST * 6);
12890 format %{ "fmuld $dst, $src1, $src2" %}
12891
12892 ins_encode %{
12893 __ fmuld(as_FloatRegister($dst$$reg),
12894 as_FloatRegister($src1$$reg),
12895 as_FloatRegister($src2$$reg));
12896 %}
12897
12898 ins_pipe(fp_dop_reg_reg_d);
12899 %}
12900
12901 // src1 * src2 + src3 (half-precision float)
12902 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12903 match(Set dst (FmaHF src3 (Binary src1 src2)));
12904 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12905 ins_encode %{
12906 assert(UseFMA, "Needs FMA instructions support.");
12907 __ fmaddh($dst$$FloatRegister,
12908 $src1$$FloatRegister,
12909 $src2$$FloatRegister,
12910 $src3$$FloatRegister);
12911 %}
12912 ins_pipe(pipe_class_default);
12913 %}
12914
12915 // src1 * src2 + src3
12916 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12917 match(Set dst (FmaF src3 (Binary src1 src2)));
12918
12919 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12920
12921 ins_encode %{
12922 assert(UseFMA, "Needs FMA instructions support.");
12923 __ fmadds(as_FloatRegister($dst$$reg),
12924 as_FloatRegister($src1$$reg),
12925 as_FloatRegister($src2$$reg),
12926 as_FloatRegister($src3$$reg));
12927 %}
12928
12929 ins_pipe(pipe_class_default);
12930 %}
12931
12932 // src1 * src2 + src3
12933 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12934 match(Set dst (FmaD src3 (Binary src1 src2)));
12935
12936 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12937
12938 ins_encode %{
12939 assert(UseFMA, "Needs FMA instructions support.");
12940 __ fmaddd(as_FloatRegister($dst$$reg),
12941 as_FloatRegister($src1$$reg),
12942 as_FloatRegister($src2$$reg),
12943 as_FloatRegister($src3$$reg));
12944 %}
12945
12946 ins_pipe(pipe_class_default);
12947 %}
12948
12949 // src1 * (-src2) + src3
12950 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12951 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12952 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12953
12954 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12955
12956 ins_encode %{
12957 assert(UseFMA, "Needs FMA instructions support.");
12958 __ fmsubs(as_FloatRegister($dst$$reg),
12959 as_FloatRegister($src1$$reg),
12960 as_FloatRegister($src2$$reg),
12961 as_FloatRegister($src3$$reg));
12962 %}
12963
12964 ins_pipe(pipe_class_default);
12965 %}
12966
12967 // src1 * (-src2) + src3
12968 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12969 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12970 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12971
12972 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12973
12974 ins_encode %{
12975 assert(UseFMA, "Needs FMA instructions support.");
12976 __ fmsubd(as_FloatRegister($dst$$reg),
12977 as_FloatRegister($src1$$reg),
12978 as_FloatRegister($src2$$reg),
12979 as_FloatRegister($src3$$reg));
12980 %}
12981
12982 ins_pipe(pipe_class_default);
12983 %}
12984
12985 // src1 * (-src2) - src3
12986 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12987 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12988 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12989
12990 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
12991
12992 ins_encode %{
12993 assert(UseFMA, "Needs FMA instructions support.");
12994 __ fnmadds(as_FloatRegister($dst$$reg),
12995 as_FloatRegister($src1$$reg),
12996 as_FloatRegister($src2$$reg),
12997 as_FloatRegister($src3$$reg));
12998 %}
12999
13000 ins_pipe(pipe_class_default);
13001 %}
13002
13003 // src1 * (-src2) - src3
13004 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13005 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13006 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13007
13008 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13009
13010 ins_encode %{
13011 assert(UseFMA, "Needs FMA instructions support.");
13012 __ fnmaddd(as_FloatRegister($dst$$reg),
13013 as_FloatRegister($src1$$reg),
13014 as_FloatRegister($src2$$reg),
13015 as_FloatRegister($src3$$reg));
13016 %}
13017
13018 ins_pipe(pipe_class_default);
13019 %}
13020
13021 // src1 * src2 - src3
13022 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13023 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13024
13025 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13026
13027 ins_encode %{
13028 assert(UseFMA, "Needs FMA instructions support.");
13029 __ fnmsubs(as_FloatRegister($dst$$reg),
13030 as_FloatRegister($src1$$reg),
13031 as_FloatRegister($src2$$reg),
13032 as_FloatRegister($src3$$reg));
13033 %}
13034
13035 ins_pipe(pipe_class_default);
13036 %}
13037
13038 // src1 * src2 - src3
13039 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13040 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13041
13042 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13043
13044 ins_encode %{
13045 assert(UseFMA, "Needs FMA instructions support.");
13046 // n.b. insn name should be fnmsubd
13047 __ fnmsub(as_FloatRegister($dst$$reg),
13048 as_FloatRegister($src1$$reg),
13049 as_FloatRegister($src2$$reg),
13050 as_FloatRegister($src3$$reg));
13051 %}
13052
13053 ins_pipe(pipe_class_default);
13054 %}
13055
13056 // Math.max(HH)H (half-precision float)
13057 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13058 match(Set dst (MaxHF src1 src2));
13059 format %{ "fmaxh $dst, $src1, $src2" %}
13060 ins_encode %{
13061 __ fmaxh($dst$$FloatRegister,
13062 $src1$$FloatRegister,
13063 $src2$$FloatRegister);
13064 %}
13065 ins_pipe(fp_dop_reg_reg_s);
13066 %}
13067
13068 // Math.min(HH)H (half-precision float)
13069 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13070 match(Set dst (MinHF src1 src2));
13071 format %{ "fminh $dst, $src1, $src2" %}
13072 ins_encode %{
13073 __ fminh($dst$$FloatRegister,
13074 $src1$$FloatRegister,
13075 $src2$$FloatRegister);
13076 %}
13077 ins_pipe(fp_dop_reg_reg_s);
13078 %}
13079
13080 // Math.max(FF)F
13081 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13082 match(Set dst (MaxF src1 src2));
13083
13084 format %{ "fmaxs $dst, $src1, $src2" %}
13085 ins_encode %{
13086 __ fmaxs(as_FloatRegister($dst$$reg),
13087 as_FloatRegister($src1$$reg),
13088 as_FloatRegister($src2$$reg));
13089 %}
13090
13091 ins_pipe(fp_dop_reg_reg_s);
13092 %}
13093
13094 // Math.min(FF)F
13095 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13096 match(Set dst (MinF src1 src2));
13097
13098 format %{ "fmins $dst, $src1, $src2" %}
13099 ins_encode %{
13100 __ fmins(as_FloatRegister($dst$$reg),
13101 as_FloatRegister($src1$$reg),
13102 as_FloatRegister($src2$$reg));
13103 %}
13104
13105 ins_pipe(fp_dop_reg_reg_s);
13106 %}
13107
13108 // Math.max(DD)D
13109 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13110 match(Set dst (MaxD src1 src2));
13111
13112 format %{ "fmaxd $dst, $src1, $src2" %}
13113 ins_encode %{
13114 __ fmaxd(as_FloatRegister($dst$$reg),
13115 as_FloatRegister($src1$$reg),
13116 as_FloatRegister($src2$$reg));
13117 %}
13118
13119 ins_pipe(fp_dop_reg_reg_d);
13120 %}
13121
13122 // Math.min(DD)D
13123 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13124 match(Set dst (MinD src1 src2));
13125
13126 format %{ "fmind $dst, $src1, $src2" %}
13127 ins_encode %{
13128 __ fmind(as_FloatRegister($dst$$reg),
13129 as_FloatRegister($src1$$reg),
13130 as_FloatRegister($src2$$reg));
13131 %}
13132
13133 ins_pipe(fp_dop_reg_reg_d);
13134 %}
13135
13136 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13137 match(Set dst (DivHF src1 src2));
13138 format %{ "fdivh $dst, $src1, $src2" %}
13139 ins_encode %{
13140 __ fdivh($dst$$FloatRegister,
13141 $src1$$FloatRegister,
13142 $src2$$FloatRegister);
13143 %}
13144 ins_pipe(fp_div_s);
13145 %}
13146
13147 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13148 match(Set dst (DivF src1 src2));
13149
13150 ins_cost(INSN_COST * 18);
13151 format %{ "fdivs $dst, $src1, $src2" %}
13152
13153 ins_encode %{
13154 __ fdivs(as_FloatRegister($dst$$reg),
13155 as_FloatRegister($src1$$reg),
13156 as_FloatRegister($src2$$reg));
13157 %}
13158
13159 ins_pipe(fp_div_s);
13160 %}
13161
13162 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13163 match(Set dst (DivD src1 src2));
13164
13165 ins_cost(INSN_COST * 32);
13166 format %{ "fdivd $dst, $src1, $src2" %}
13167
13168 ins_encode %{
13169 __ fdivd(as_FloatRegister($dst$$reg),
13170 as_FloatRegister($src1$$reg),
13171 as_FloatRegister($src2$$reg));
13172 %}
13173
13174 ins_pipe(fp_div_d);
13175 %}
13176
13177 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13178 match(Set dst (NegF src));
13179
13180 ins_cost(INSN_COST * 3);
13181 format %{ "fneg $dst, $src" %}
13182
13183 ins_encode %{
13184 __ fnegs(as_FloatRegister($dst$$reg),
13185 as_FloatRegister($src$$reg));
13186 %}
13187
13188 ins_pipe(fp_uop_s);
13189 %}
13190
13191 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13192 match(Set dst (NegD src));
13193
13194 ins_cost(INSN_COST * 3);
13195 format %{ "fnegd $dst, $src" %}
13196
13197 ins_encode %{
13198 __ fnegd(as_FloatRegister($dst$$reg),
13199 as_FloatRegister($src$$reg));
13200 %}
13201
13202 ins_pipe(fp_uop_d);
13203 %}
13204
13205 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13206 %{
13207 match(Set dst (AbsI src));
13208
13209 effect(KILL cr);
13210 ins_cost(INSN_COST * 2);
13211 format %{ "cmpw $src, zr\n\t"
13212 "cnegw $dst, $src, Assembler::LT\t# int abs"
13213 %}
13214
13215 ins_encode %{
13216 __ cmpw(as_Register($src$$reg), zr);
13217 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13218 %}
13219 ins_pipe(pipe_class_default);
13220 %}
13221
13222 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13223 %{
13224 match(Set dst (AbsL src));
13225
13226 effect(KILL cr);
13227 ins_cost(INSN_COST * 2);
13228 format %{ "cmp $src, zr\n\t"
13229 "cneg $dst, $src, Assembler::LT\t# long abs"
13230 %}
13231
13232 ins_encode %{
13233 __ cmp(as_Register($src$$reg), zr);
13234 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13235 %}
13236 ins_pipe(pipe_class_default);
13237 %}
13238
13239 instruct absF_reg(vRegF dst, vRegF src) %{
13240 match(Set dst (AbsF src));
13241
13242 ins_cost(INSN_COST * 3);
13243 format %{ "fabss $dst, $src" %}
13244 ins_encode %{
13245 __ fabss(as_FloatRegister($dst$$reg),
13246 as_FloatRegister($src$$reg));
13247 %}
13248
13249 ins_pipe(fp_uop_s);
13250 %}
13251
13252 instruct absD_reg(vRegD dst, vRegD src) %{
13253 match(Set dst (AbsD src));
13254
13255 ins_cost(INSN_COST * 3);
13256 format %{ "fabsd $dst, $src" %}
13257 ins_encode %{
13258 __ fabsd(as_FloatRegister($dst$$reg),
13259 as_FloatRegister($src$$reg));
13260 %}
13261
13262 ins_pipe(fp_uop_d);
13263 %}
13264
13265 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13266 match(Set dst (AbsF (SubF src1 src2)));
13267
13268 ins_cost(INSN_COST * 3);
13269 format %{ "fabds $dst, $src1, $src2" %}
13270 ins_encode %{
13271 __ fabds(as_FloatRegister($dst$$reg),
13272 as_FloatRegister($src1$$reg),
13273 as_FloatRegister($src2$$reg));
13274 %}
13275
13276 ins_pipe(fp_uop_s);
13277 %}
13278
13279 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13280 match(Set dst (AbsD (SubD src1 src2)));
13281
13282 ins_cost(INSN_COST * 3);
13283 format %{ "fabdd $dst, $src1, $src2" %}
13284 ins_encode %{
13285 __ fabdd(as_FloatRegister($dst$$reg),
13286 as_FloatRegister($src1$$reg),
13287 as_FloatRegister($src2$$reg));
13288 %}
13289
13290 ins_pipe(fp_uop_d);
13291 %}
13292
13293 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13294 match(Set dst (SqrtD src));
13295
13296 ins_cost(INSN_COST * 50);
13297 format %{ "fsqrtd $dst, $src" %}
13298 ins_encode %{
13299 __ fsqrtd(as_FloatRegister($dst$$reg),
13300 as_FloatRegister($src$$reg));
13301 %}
13302
13303 ins_pipe(fp_div_s);
13304 %}
13305
13306 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13307 match(Set dst (SqrtF src));
13308
13309 ins_cost(INSN_COST * 50);
13310 format %{ "fsqrts $dst, $src" %}
13311 ins_encode %{
13312 __ fsqrts(as_FloatRegister($dst$$reg),
13313 as_FloatRegister($src$$reg));
13314 %}
13315
13316 ins_pipe(fp_div_d);
13317 %}
13318
13319 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13320 match(Set dst (SqrtHF src));
13321 format %{ "fsqrth $dst, $src" %}
13322 ins_encode %{
13323 __ fsqrth($dst$$FloatRegister,
13324 $src$$FloatRegister);
13325 %}
13326 ins_pipe(fp_div_s);
13327 %}
13328
13329 // Math.rint, floor, ceil
13330 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13331 match(Set dst (RoundDoubleMode src rmode));
13332 format %{ "frint $dst, $src, $rmode" %}
13333 ins_encode %{
13334 switch ($rmode$$constant) {
13335 case RoundDoubleModeNode::rmode_rint:
13336 __ frintnd(as_FloatRegister($dst$$reg),
13337 as_FloatRegister($src$$reg));
13338 break;
13339 case RoundDoubleModeNode::rmode_floor:
13340 __ frintmd(as_FloatRegister($dst$$reg),
13341 as_FloatRegister($src$$reg));
13342 break;
13343 case RoundDoubleModeNode::rmode_ceil:
13344 __ frintpd(as_FloatRegister($dst$$reg),
13345 as_FloatRegister($src$$reg));
13346 break;
13347 }
13348 %}
13349 ins_pipe(fp_uop_d);
13350 %}
13351
13352 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13353 match(Set dst (CopySignD src1 (Binary src2 zero)));
13354 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13355 format %{ "CopySignD $dst $src1 $src2" %}
13356 ins_encode %{
13357 FloatRegister dst = as_FloatRegister($dst$$reg),
13358 src1 = as_FloatRegister($src1$$reg),
13359 src2 = as_FloatRegister($src2$$reg),
13360 zero = as_FloatRegister($zero$$reg);
13361 __ fnegd(dst, zero);
13362 __ bsl(dst, __ T8B, src2, src1);
13363 %}
13364 ins_pipe(fp_uop_d);
13365 %}
13366
13367 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13368 match(Set dst (CopySignF src1 src2));
13369 effect(TEMP_DEF dst, USE src1, USE src2);
13370 format %{ "CopySignF $dst $src1 $src2" %}
13371 ins_encode %{
13372 FloatRegister dst = as_FloatRegister($dst$$reg),
13373 src1 = as_FloatRegister($src1$$reg),
13374 src2 = as_FloatRegister($src2$$reg);
13375 __ movi(dst, __ T2S, 0x80, 24);
13376 __ bsl(dst, __ T8B, src2, src1);
13377 %}
13378 ins_pipe(fp_uop_d);
13379 %}
13380
13381 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13382 match(Set dst (SignumD src (Binary zero one)));
13383 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13384 format %{ "signumD $dst, $src" %}
13385 ins_encode %{
13386 FloatRegister src = as_FloatRegister($src$$reg),
13387 dst = as_FloatRegister($dst$$reg),
13388 zero = as_FloatRegister($zero$$reg),
13389 one = as_FloatRegister($one$$reg);
13390 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13391 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13392 // Bit selection instruction gets bit from "one" for each enabled bit in
13393 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13394 // NaN the whole "src" will be copied because "dst" is zero. For all other
13395 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13396 // from "src", and all other bits are copied from 1.0.
13397 __ bsl(dst, __ T8B, one, src);
13398 %}
13399 ins_pipe(fp_uop_d);
13400 %}
13401
13402 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13403 match(Set dst (SignumF src (Binary zero one)));
13404 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13405 format %{ "signumF $dst, $src" %}
13406 ins_encode %{
13407 FloatRegister src = as_FloatRegister($src$$reg),
13408 dst = as_FloatRegister($dst$$reg),
13409 zero = as_FloatRegister($zero$$reg),
13410 one = as_FloatRegister($one$$reg);
13411 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13412 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13413 // Bit selection instruction gets bit from "one" for each enabled bit in
13414 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13415 // NaN the whole "src" will be copied because "dst" is zero. For all other
13416 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13417 // from "src", and all other bits are copied from 1.0.
13418 __ bsl(dst, __ T8B, one, src);
13419 %}
13420 ins_pipe(fp_uop_d);
13421 %}
13422
13423 instruct onspinwait() %{
13424 match(OnSpinWait);
13425 ins_cost(INSN_COST);
13426
13427 format %{ "onspinwait" %}
13428
13429 ins_encode %{
13430 __ spin_wait();
13431 %}
13432 ins_pipe(pipe_class_empty);
13433 %}
13434
13435 // ============================================================================
13436 // Logical Instructions
13437
13438 // Integer Logical Instructions
13439
13440 // And Instructions
13441
13442
13443 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13444 match(Set dst (AndI src1 src2));
13445
13446 format %{ "andw $dst, $src1, $src2\t# int" %}
13447
13448 ins_cost(INSN_COST);
13449 ins_encode %{
13450 __ andw(as_Register($dst$$reg),
13451 as_Register($src1$$reg),
13452 as_Register($src2$$reg));
13453 %}
13454
13455 ins_pipe(ialu_reg_reg);
13456 %}
13457
13458 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13459 match(Set dst (AndI src1 src2));
13460
13461 format %{ "andsw $dst, $src1, $src2\t# int" %}
13462
13463 ins_cost(INSN_COST);
13464 ins_encode %{
13465 __ andw(as_Register($dst$$reg),
13466 as_Register($src1$$reg),
13467 (uint64_t)($src2$$constant));
13468 %}
13469
13470 ins_pipe(ialu_reg_imm);
13471 %}
13472
13473 // Or Instructions
13474
13475 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13476 match(Set dst (OrI src1 src2));
13477
13478 format %{ "orrw $dst, $src1, $src2\t# int" %}
13479
13480 ins_cost(INSN_COST);
13481 ins_encode %{
13482 __ orrw(as_Register($dst$$reg),
13483 as_Register($src1$$reg),
13484 as_Register($src2$$reg));
13485 %}
13486
13487 ins_pipe(ialu_reg_reg);
13488 %}
13489
13490 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13491 match(Set dst (OrI src1 src2));
13492
13493 format %{ "orrw $dst, $src1, $src2\t# int" %}
13494
13495 ins_cost(INSN_COST);
13496 ins_encode %{
13497 __ orrw(as_Register($dst$$reg),
13498 as_Register($src1$$reg),
13499 (uint64_t)($src2$$constant));
13500 %}
13501
13502 ins_pipe(ialu_reg_imm);
13503 %}
13504
13505 // Xor Instructions
13506
13507 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13508 match(Set dst (XorI src1 src2));
13509
13510 format %{ "eorw $dst, $src1, $src2\t# int" %}
13511
13512 ins_cost(INSN_COST);
13513 ins_encode %{
13514 __ eorw(as_Register($dst$$reg),
13515 as_Register($src1$$reg),
13516 as_Register($src2$$reg));
13517 %}
13518
13519 ins_pipe(ialu_reg_reg);
13520 %}
13521
13522 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13523 match(Set dst (XorI src1 src2));
13524
13525 format %{ "eorw $dst, $src1, $src2\t# int" %}
13526
13527 ins_cost(INSN_COST);
13528 ins_encode %{
13529 __ eorw(as_Register($dst$$reg),
13530 as_Register($src1$$reg),
13531 (uint64_t)($src2$$constant));
13532 %}
13533
13534 ins_pipe(ialu_reg_imm);
13535 %}
13536
13537 // Long Logical Instructions
13538 // TODO
13539
13540 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13541 match(Set dst (AndL src1 src2));
13542
13543 format %{ "and $dst, $src1, $src2\t# int" %}
13544
13545 ins_cost(INSN_COST);
13546 ins_encode %{
13547 __ andr(as_Register($dst$$reg),
13548 as_Register($src1$$reg),
13549 as_Register($src2$$reg));
13550 %}
13551
13552 ins_pipe(ialu_reg_reg);
13553 %}
13554
13555 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13556 match(Set dst (AndL src1 src2));
13557
13558 format %{ "and $dst, $src1, $src2\t# int" %}
13559
13560 ins_cost(INSN_COST);
13561 ins_encode %{
13562 __ andr(as_Register($dst$$reg),
13563 as_Register($src1$$reg),
13564 (uint64_t)($src2$$constant));
13565 %}
13566
13567 ins_pipe(ialu_reg_imm);
13568 %}
13569
13570 // Or Instructions
13571
13572 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13573 match(Set dst (OrL src1 src2));
13574
13575 format %{ "orr $dst, $src1, $src2\t# int" %}
13576
13577 ins_cost(INSN_COST);
13578 ins_encode %{
13579 __ orr(as_Register($dst$$reg),
13580 as_Register($src1$$reg),
13581 as_Register($src2$$reg));
13582 %}
13583
13584 ins_pipe(ialu_reg_reg);
13585 %}
13586
13587 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13588 match(Set dst (OrL src1 src2));
13589
13590 format %{ "orr $dst, $src1, $src2\t# int" %}
13591
13592 ins_cost(INSN_COST);
13593 ins_encode %{
13594 __ orr(as_Register($dst$$reg),
13595 as_Register($src1$$reg),
13596 (uint64_t)($src2$$constant));
13597 %}
13598
13599 ins_pipe(ialu_reg_imm);
13600 %}
13601
13602 // Xor Instructions
13603
13604 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13605 match(Set dst (XorL src1 src2));
13606
13607 format %{ "eor $dst, $src1, $src2\t# int" %}
13608
13609 ins_cost(INSN_COST);
13610 ins_encode %{
13611 __ eor(as_Register($dst$$reg),
13612 as_Register($src1$$reg),
13613 as_Register($src2$$reg));
13614 %}
13615
13616 ins_pipe(ialu_reg_reg);
13617 %}
13618
13619 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13620 match(Set dst (XorL src1 src2));
13621
13622 ins_cost(INSN_COST);
13623 format %{ "eor $dst, $src1, $src2\t# int" %}
13624
13625 ins_encode %{
13626 __ eor(as_Register($dst$$reg),
13627 as_Register($src1$$reg),
13628 (uint64_t)($src2$$constant));
13629 %}
13630
13631 ins_pipe(ialu_reg_imm);
13632 %}
13633
13634 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13635 %{
13636 match(Set dst (ConvI2L src));
13637
13638 ins_cost(INSN_COST);
13639 format %{ "sxtw $dst, $src\t# i2l" %}
13640 ins_encode %{
13641 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13642 %}
13643 ins_pipe(ialu_reg_shift);
13644 %}
13645
13646 // this pattern occurs in bigmath arithmetic
13647 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13648 %{
13649 match(Set dst (AndL (ConvI2L src) mask));
13650
13651 ins_cost(INSN_COST);
13652 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13653 ins_encode %{
13654 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13655 %}
13656
13657 ins_pipe(ialu_reg_shift);
13658 %}
13659
13660 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13661 match(Set dst (ConvL2I src));
13662
13663 ins_cost(INSN_COST);
13664 format %{ "movw $dst, $src \t// l2i" %}
13665
13666 ins_encode %{
13667 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13668 %}
13669
13670 ins_pipe(ialu_reg);
13671 %}
13672
13673 instruct convD2F_reg(vRegF dst, vRegD src) %{
13674 match(Set dst (ConvD2F src));
13675
13676 ins_cost(INSN_COST * 5);
13677 format %{ "fcvtd $dst, $src \t// d2f" %}
13678
13679 ins_encode %{
13680 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13681 %}
13682
13683 ins_pipe(fp_d2f);
13684 %}
13685
13686 instruct convF2D_reg(vRegD dst, vRegF src) %{
13687 match(Set dst (ConvF2D src));
13688
13689 ins_cost(INSN_COST * 5);
13690 format %{ "fcvts $dst, $src \t// f2d" %}
13691
13692 ins_encode %{
13693 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13694 %}
13695
13696 ins_pipe(fp_f2d);
13697 %}
13698
13699 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13700 match(Set dst (ConvF2I src));
13701
13702 ins_cost(INSN_COST * 5);
13703 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13704
13705 ins_encode %{
13706 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13707 %}
13708
13709 ins_pipe(fp_f2i);
13710 %}
13711
13712 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13713 match(Set dst (ConvF2L src));
13714
13715 ins_cost(INSN_COST * 5);
13716 format %{ "fcvtzs $dst, $src \t// f2l" %}
13717
13718 ins_encode %{
13719 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13720 %}
13721
13722 ins_pipe(fp_f2l);
13723 %}
13724
13725 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13726 match(Set dst (ConvF2HF src));
13727 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13728 "smov $dst, $tmp\t# move result from $tmp to $dst"
13729 %}
13730 effect(TEMP tmp);
13731 ins_encode %{
13732 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13733 %}
13734 ins_pipe(pipe_slow);
13735 %}
13736
13737 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13738 match(Set dst (ConvHF2F src));
13739 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13740 "fcvt $dst, $tmp\t# convert half to single precision"
13741 %}
13742 effect(TEMP tmp);
13743 ins_encode %{
13744 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13745 %}
13746 ins_pipe(pipe_slow);
13747 %}
13748
13749 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13750 match(Set dst (ConvI2F src));
13751
13752 ins_cost(INSN_COST * 5);
13753 format %{ "scvtfws $dst, $src \t// i2f" %}
13754
13755 ins_encode %{
13756 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13757 %}
13758
13759 ins_pipe(fp_i2f);
13760 %}
13761
13762 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13763 match(Set dst (ConvL2F src));
13764
13765 ins_cost(INSN_COST * 5);
13766 format %{ "scvtfs $dst, $src \t// l2f" %}
13767
13768 ins_encode %{
13769 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13770 %}
13771
13772 ins_pipe(fp_l2f);
13773 %}
13774
13775 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13776 match(Set dst (ConvD2I src));
13777
13778 ins_cost(INSN_COST * 5);
13779 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13780
13781 ins_encode %{
13782 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13783 %}
13784
13785 ins_pipe(fp_d2i);
13786 %}
13787
13788 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13789 match(Set dst (ConvD2L src));
13790
13791 ins_cost(INSN_COST * 5);
13792 format %{ "fcvtzd $dst, $src \t// d2l" %}
13793
13794 ins_encode %{
13795 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13796 %}
13797
13798 ins_pipe(fp_d2l);
13799 %}
13800
13801 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13802 match(Set dst (ConvI2D src));
13803
13804 ins_cost(INSN_COST * 5);
13805 format %{ "scvtfwd $dst, $src \t// i2d" %}
13806
13807 ins_encode %{
13808 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13809 %}
13810
13811 ins_pipe(fp_i2d);
13812 %}
13813
13814 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13815 match(Set dst (ConvL2D src));
13816
13817 ins_cost(INSN_COST * 5);
13818 format %{ "scvtfd $dst, $src \t// l2d" %}
13819
13820 ins_encode %{
13821 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13822 %}
13823
13824 ins_pipe(fp_l2d);
13825 %}
13826
13827 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13828 %{
13829 match(Set dst (RoundD src));
13830 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13831 format %{ "java_round_double $dst,$src"%}
13832 ins_encode %{
13833 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13834 as_FloatRegister($ftmp$$reg));
13835 %}
13836 ins_pipe(pipe_slow);
13837 %}
13838
13839 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13840 %{
13841 match(Set dst (RoundF src));
13842 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13843 format %{ "java_round_float $dst,$src"%}
13844 ins_encode %{
13845 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13846 as_FloatRegister($ftmp$$reg));
13847 %}
13848 ins_pipe(pipe_slow);
13849 %}
13850
13851 // stack <-> reg and reg <-> reg shuffles with no conversion
13852
13853 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13854
13855 match(Set dst (MoveF2I src));
13856
13857 effect(DEF dst, USE src);
13858
13859 ins_cost(4 * INSN_COST);
13860
13861 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13862
13863 ins_encode %{
13864 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13865 %}
13866
13867 ins_pipe(iload_reg_reg);
13868
13869 %}
13870
13871 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13872
13873 match(Set dst (MoveI2F src));
13874
13875 effect(DEF dst, USE src);
13876
13877 ins_cost(4 * INSN_COST);
13878
13879 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13880
13881 ins_encode %{
13882 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13883 %}
13884
13885 ins_pipe(pipe_class_memory);
13886
13887 %}
13888
13889 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13890
13891 match(Set dst (MoveD2L src));
13892
13893 effect(DEF dst, USE src);
13894
13895 ins_cost(4 * INSN_COST);
13896
13897 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13898
13899 ins_encode %{
13900 __ ldr($dst$$Register, Address(sp, $src$$disp));
13901 %}
13902
13903 ins_pipe(iload_reg_reg);
13904
13905 %}
13906
13907 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13908
13909 match(Set dst (MoveL2D src));
13910
13911 effect(DEF dst, USE src);
13912
13913 ins_cost(4 * INSN_COST);
13914
13915 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13916
13917 ins_encode %{
13918 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13919 %}
13920
13921 ins_pipe(pipe_class_memory);
13922
13923 %}
13924
13925 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13926
13927 match(Set dst (MoveF2I src));
13928
13929 effect(DEF dst, USE src);
13930
13931 ins_cost(INSN_COST);
13932
13933 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13934
13935 ins_encode %{
13936 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13937 %}
13938
13939 ins_pipe(pipe_class_memory);
13940
13941 %}
13942
13943 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13944
13945 match(Set dst (MoveI2F src));
13946
13947 effect(DEF dst, USE src);
13948
13949 ins_cost(INSN_COST);
13950
13951 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13952
13953 ins_encode %{
13954 __ strw($src$$Register, Address(sp, $dst$$disp));
13955 %}
13956
13957 ins_pipe(istore_reg_reg);
13958
13959 %}
13960
13961 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13962
13963 match(Set dst (MoveD2L src));
13964
13965 effect(DEF dst, USE src);
13966
13967 ins_cost(INSN_COST);
13968
13969 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13970
13971 ins_encode %{
13972 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13973 %}
13974
13975 ins_pipe(pipe_class_memory);
13976
13977 %}
13978
13979 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13980
13981 match(Set dst (MoveL2D src));
13982
13983 effect(DEF dst, USE src);
13984
13985 ins_cost(INSN_COST);
13986
13987 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13988
13989 ins_encode %{
13990 __ str($src$$Register, Address(sp, $dst$$disp));
13991 %}
13992
13993 ins_pipe(istore_reg_reg);
13994
13995 %}
13996
13997 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13998
13999 match(Set dst (MoveF2I src));
14000
14001 effect(DEF dst, USE src);
14002
14003 ins_cost(INSN_COST);
14004
14005 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14006
14007 ins_encode %{
14008 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14009 %}
14010
14011 ins_pipe(fp_f2i);
14012
14013 %}
14014
14015 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14016
14017 match(Set dst (MoveI2F src));
14018
14019 effect(DEF dst, USE src);
14020
14021 ins_cost(INSN_COST);
14022
14023 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14024
14025 ins_encode %{
14026 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14027 %}
14028
14029 ins_pipe(fp_i2f);
14030
14031 %}
14032
14033 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14034
14035 match(Set dst (MoveD2L src));
14036
14037 effect(DEF dst, USE src);
14038
14039 ins_cost(INSN_COST);
14040
14041 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14042
14043 ins_encode %{
14044 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14045 %}
14046
14047 ins_pipe(fp_d2l);
14048
14049 %}
14050
14051 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14052
14053 match(Set dst (MoveL2D src));
14054
14055 effect(DEF dst, USE src);
14056
14057 ins_cost(INSN_COST);
14058
14059 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14060
14061 ins_encode %{
14062 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14063 %}
14064
14065 ins_pipe(fp_l2d);
14066
14067 %}
14068
14069 // ============================================================================
14070 // clearing of an array
14071
14072 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14073 %{
14074 match(Set dummy (ClearArray cnt base));
14075 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14076
14077 ins_cost(4 * INSN_COST);
14078 format %{ "ClearArray $cnt, $base" %}
14079
14080 ins_encode %{
14081 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14082 if (tpc == nullptr) {
14083 ciEnv::current()->record_failure("CodeCache is full");
14084 return;
14085 }
14086 %}
14087
14088 ins_pipe(pipe_class_memory);
14089 %}
14090
14091 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14092 %{
14093 predicate((uint64_t)n->in(2)->get_long()
14094 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14095 match(Set dummy (ClearArray cnt base));
14096 effect(TEMP temp, USE_KILL base, KILL cr);
14097
14098 ins_cost(4 * INSN_COST);
14099 format %{ "ClearArray $cnt, $base" %}
14100
14101 ins_encode %{
14102 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14103 if (tpc == nullptr) {
14104 ciEnv::current()->record_failure("CodeCache is full");
14105 return;
14106 }
14107 %}
14108
14109 ins_pipe(pipe_class_memory);
14110 %}
14111
14112 // ============================================================================
14113 // Overflow Math Instructions
14114
14115 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14116 %{
14117 match(Set cr (OverflowAddI op1 op2));
14118
14119 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14120 ins_cost(INSN_COST);
14121 ins_encode %{
14122 __ cmnw($op1$$Register, $op2$$Register);
14123 %}
14124
14125 ins_pipe(icmp_reg_reg);
14126 %}
14127
14128 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14129 %{
14130 match(Set cr (OverflowAddI op1 op2));
14131
14132 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14133 ins_cost(INSN_COST);
14134 ins_encode %{
14135 __ cmnw($op1$$Register, $op2$$constant);
14136 %}
14137
14138 ins_pipe(icmp_reg_imm);
14139 %}
14140
14141 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14142 %{
14143 match(Set cr (OverflowAddL op1 op2));
14144
14145 format %{ "cmn $op1, $op2\t# overflow check long" %}
14146 ins_cost(INSN_COST);
14147 ins_encode %{
14148 __ cmn($op1$$Register, $op2$$Register);
14149 %}
14150
14151 ins_pipe(icmp_reg_reg);
14152 %}
14153
14154 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14155 %{
14156 match(Set cr (OverflowAddL op1 op2));
14157
14158 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14159 ins_cost(INSN_COST);
14160 ins_encode %{
14161 __ adds(zr, $op1$$Register, $op2$$constant);
14162 %}
14163
14164 ins_pipe(icmp_reg_imm);
14165 %}
14166
14167 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14168 %{
14169 match(Set cr (OverflowSubI op1 op2));
14170
14171 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14172 ins_cost(INSN_COST);
14173 ins_encode %{
14174 __ cmpw($op1$$Register, $op2$$Register);
14175 %}
14176
14177 ins_pipe(icmp_reg_reg);
14178 %}
14179
14180 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14181 %{
14182 match(Set cr (OverflowSubI op1 op2));
14183
14184 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14185 ins_cost(INSN_COST);
14186 ins_encode %{
14187 __ cmpw($op1$$Register, $op2$$constant);
14188 %}
14189
14190 ins_pipe(icmp_reg_imm);
14191 %}
14192
14193 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14194 %{
14195 match(Set cr (OverflowSubL op1 op2));
14196
14197 format %{ "cmp $op1, $op2\t# overflow check long" %}
14198 ins_cost(INSN_COST);
14199 ins_encode %{
14200 __ cmp($op1$$Register, $op2$$Register);
14201 %}
14202
14203 ins_pipe(icmp_reg_reg);
14204 %}
14205
14206 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14207 %{
14208 match(Set cr (OverflowSubL op1 op2));
14209
14210 format %{ "cmp $op1, $op2\t# overflow check long" %}
14211 ins_cost(INSN_COST);
14212 ins_encode %{
14213 __ subs(zr, $op1$$Register, $op2$$constant);
14214 %}
14215
14216 ins_pipe(icmp_reg_imm);
14217 %}
14218
14219 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14220 %{
14221 match(Set cr (OverflowSubI zero op1));
14222
14223 format %{ "cmpw zr, $op1\t# overflow check int" %}
14224 ins_cost(INSN_COST);
14225 ins_encode %{
14226 __ cmpw(zr, $op1$$Register);
14227 %}
14228
14229 ins_pipe(icmp_reg_imm);
14230 %}
14231
14232 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14233 %{
14234 match(Set cr (OverflowSubL zero op1));
14235
14236 format %{ "cmp zr, $op1\t# overflow check long" %}
14237 ins_cost(INSN_COST);
14238 ins_encode %{
14239 __ cmp(zr, $op1$$Register);
14240 %}
14241
14242 ins_pipe(icmp_reg_imm);
14243 %}
14244
14245 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14246 %{
14247 match(Set cr (OverflowMulI op1 op2));
14248
14249 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14250 "cmp rscratch1, rscratch1, sxtw\n\t"
14251 "movw rscratch1, #0x80000000\n\t"
14252 "cselw rscratch1, rscratch1, zr, NE\n\t"
14253 "cmpw rscratch1, #1" %}
14254 ins_cost(5 * INSN_COST);
14255 ins_encode %{
14256 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14257 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14258 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14259 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14260 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14261 %}
14262
14263 ins_pipe(pipe_slow);
14264 %}
14265
14266 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14267 %{
14268 match(If cmp (OverflowMulI op1 op2));
14269 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14270 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14271 effect(USE labl, KILL cr);
14272
14273 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14274 "cmp rscratch1, rscratch1, sxtw\n\t"
14275 "b$cmp $labl" %}
14276 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14277 ins_encode %{
14278 Label* L = $labl$$label;
14279 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14280 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14281 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14282 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14283 %}
14284
14285 ins_pipe(pipe_serial);
14286 %}
14287
14288 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14289 %{
14290 match(Set cr (OverflowMulL op1 op2));
14291
14292 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14293 "smulh rscratch2, $op1, $op2\n\t"
14294 "cmp rscratch2, rscratch1, ASR #63\n\t"
14295 "movw rscratch1, #0x80000000\n\t"
14296 "cselw rscratch1, rscratch1, zr, NE\n\t"
14297 "cmpw rscratch1, #1" %}
14298 ins_cost(6 * INSN_COST);
14299 ins_encode %{
14300 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14301 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14302 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14303 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14304 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14305 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14306 %}
14307
14308 ins_pipe(pipe_slow);
14309 %}
14310
14311 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14312 %{
14313 match(If cmp (OverflowMulL op1 op2));
14314 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14315 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14316 effect(USE labl, KILL cr);
14317
14318 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14319 "smulh rscratch2, $op1, $op2\n\t"
14320 "cmp rscratch2, rscratch1, ASR #63\n\t"
14321 "b$cmp $labl" %}
14322 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14323 ins_encode %{
14324 Label* L = $labl$$label;
14325 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14326 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14327 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14328 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14329 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14330 %}
14331
14332 ins_pipe(pipe_serial);
14333 %}
14334
14335 // ============================================================================
14336 // Compare Instructions
14337
14338 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14339 %{
14340 match(Set cr (CmpI op1 op2));
14341
14342 effect(DEF cr, USE op1, USE op2);
14343
14344 ins_cost(INSN_COST);
14345 format %{ "cmpw $op1, $op2" %}
14346
14347 ins_encode(aarch64_enc_cmpw(op1, op2));
14348
14349 ins_pipe(icmp_reg_reg);
14350 %}
14351
14352 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14353 %{
14354 match(Set cr (CmpI op1 zero));
14355
14356 effect(DEF cr, USE op1);
14357
14358 ins_cost(INSN_COST);
14359 format %{ "cmpw $op1, 0" %}
14360
14361 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14362
14363 ins_pipe(icmp_reg_imm);
14364 %}
14365
14366 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14367 %{
14368 match(Set cr (CmpI op1 op2));
14369
14370 effect(DEF cr, USE op1);
14371
14372 ins_cost(INSN_COST);
14373 format %{ "cmpw $op1, $op2" %}
14374
14375 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14376
14377 ins_pipe(icmp_reg_imm);
14378 %}
14379
14380 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14381 %{
14382 match(Set cr (CmpI op1 op2));
14383
14384 effect(DEF cr, USE op1);
14385
14386 ins_cost(INSN_COST * 2);
14387 format %{ "cmpw $op1, $op2" %}
14388
14389 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14390
14391 ins_pipe(icmp_reg_imm);
14392 %}
14393
14394 // Unsigned compare Instructions; really, same as signed compare
14395 // except it should only be used to feed an If or a CMovI which takes a
14396 // cmpOpU.
14397
14398 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14399 %{
14400 match(Set cr (CmpU op1 op2));
14401
14402 effect(DEF cr, USE op1, USE op2);
14403
14404 ins_cost(INSN_COST);
14405 format %{ "cmpw $op1, $op2\t# unsigned" %}
14406
14407 ins_encode(aarch64_enc_cmpw(op1, op2));
14408
14409 ins_pipe(icmp_reg_reg);
14410 %}
14411
14412 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14413 %{
14414 match(Set cr (CmpU op1 zero));
14415
14416 effect(DEF cr, USE op1);
14417
14418 ins_cost(INSN_COST);
14419 format %{ "cmpw $op1, #0\t# unsigned" %}
14420
14421 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14422
14423 ins_pipe(icmp_reg_imm);
14424 %}
14425
14426 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14427 %{
14428 match(Set cr (CmpU op1 op2));
14429
14430 effect(DEF cr, USE op1);
14431
14432 ins_cost(INSN_COST);
14433 format %{ "cmpw $op1, $op2\t# unsigned" %}
14434
14435 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14436
14437 ins_pipe(icmp_reg_imm);
14438 %}
14439
14440 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14441 %{
14442 match(Set cr (CmpU op1 op2));
14443
14444 effect(DEF cr, USE op1);
14445
14446 ins_cost(INSN_COST * 2);
14447 format %{ "cmpw $op1, $op2\t# unsigned" %}
14448
14449 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14450
14451 ins_pipe(icmp_reg_imm);
14452 %}
14453
14454 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14455 %{
14456 match(Set cr (CmpL op1 op2));
14457
14458 effect(DEF cr, USE op1, USE op2);
14459
14460 ins_cost(INSN_COST);
14461 format %{ "cmp $op1, $op2" %}
14462
14463 ins_encode(aarch64_enc_cmp(op1, op2));
14464
14465 ins_pipe(icmp_reg_reg);
14466 %}
14467
14468 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14469 %{
14470 match(Set cr (CmpL op1 zero));
14471
14472 effect(DEF cr, USE op1);
14473
14474 ins_cost(INSN_COST);
14475 format %{ "tst $op1" %}
14476
14477 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14478
14479 ins_pipe(icmp_reg_imm);
14480 %}
14481
14482 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14483 %{
14484 match(Set cr (CmpL op1 op2));
14485
14486 effect(DEF cr, USE op1);
14487
14488 ins_cost(INSN_COST);
14489 format %{ "cmp $op1, $op2" %}
14490
14491 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14492
14493 ins_pipe(icmp_reg_imm);
14494 %}
14495
14496 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14497 %{
14498 match(Set cr (CmpL op1 op2));
14499
14500 effect(DEF cr, USE op1);
14501
14502 ins_cost(INSN_COST * 2);
14503 format %{ "cmp $op1, $op2" %}
14504
14505 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14506
14507 ins_pipe(icmp_reg_imm);
14508 %}
14509
14510 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14511 %{
14512 match(Set cr (CmpUL op1 op2));
14513
14514 effect(DEF cr, USE op1, USE op2);
14515
14516 ins_cost(INSN_COST);
14517 format %{ "cmp $op1, $op2" %}
14518
14519 ins_encode(aarch64_enc_cmp(op1, op2));
14520
14521 ins_pipe(icmp_reg_reg);
14522 %}
14523
14524 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14525 %{
14526 match(Set cr (CmpUL op1 zero));
14527
14528 effect(DEF cr, USE op1);
14529
14530 ins_cost(INSN_COST);
14531 format %{ "tst $op1" %}
14532
14533 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14534
14535 ins_pipe(icmp_reg_imm);
14536 %}
14537
14538 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14539 %{
14540 match(Set cr (CmpUL op1 op2));
14541
14542 effect(DEF cr, USE op1);
14543
14544 ins_cost(INSN_COST);
14545 format %{ "cmp $op1, $op2" %}
14546
14547 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14548
14549 ins_pipe(icmp_reg_imm);
14550 %}
14551
14552 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14553 %{
14554 match(Set cr (CmpUL op1 op2));
14555
14556 effect(DEF cr, USE op1);
14557
14558 ins_cost(INSN_COST * 2);
14559 format %{ "cmp $op1, $op2" %}
14560
14561 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14562
14563 ins_pipe(icmp_reg_imm);
14564 %}
14565
14566 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14567 %{
14568 match(Set cr (CmpP op1 op2));
14569
14570 effect(DEF cr, USE op1, USE op2);
14571
14572 ins_cost(INSN_COST);
14573 format %{ "cmp $op1, $op2\t // ptr" %}
14574
14575 ins_encode(aarch64_enc_cmpp(op1, op2));
14576
14577 ins_pipe(icmp_reg_reg);
14578 %}
14579
14580 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14581 %{
14582 match(Set cr (CmpN op1 op2));
14583
14584 effect(DEF cr, USE op1, USE op2);
14585
14586 ins_cost(INSN_COST);
14587 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14588
14589 ins_encode(aarch64_enc_cmpn(op1, op2));
14590
14591 ins_pipe(icmp_reg_reg);
14592 %}
14593
14594 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14595 %{
14596 match(Set cr (CmpP op1 zero));
14597
14598 effect(DEF cr, USE op1, USE zero);
14599
14600 ins_cost(INSN_COST);
14601 format %{ "cmp $op1, 0\t // ptr" %}
14602
14603 ins_encode(aarch64_enc_testp(op1));
14604
14605 ins_pipe(icmp_reg_imm);
14606 %}
14607
14608 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14609 %{
14610 match(Set cr (CmpN op1 zero));
14611
14612 effect(DEF cr, USE op1, USE zero);
14613
14614 ins_cost(INSN_COST);
14615 format %{ "cmp $op1, 0\t // compressed ptr" %}
14616
14617 ins_encode(aarch64_enc_testn(op1));
14618
14619 ins_pipe(icmp_reg_imm);
14620 %}
14621
14622 // FP comparisons
14623 //
14624 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14625 // using normal cmpOp. See declaration of rFlagsReg for details.
14626
14627 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14628 %{
14629 match(Set cr (CmpF src1 src2));
14630
14631 ins_cost(3 * INSN_COST);
14632 format %{ "fcmps $src1, $src2" %}
14633
14634 ins_encode %{
14635 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14636 %}
14637
14638 ins_pipe(pipe_class_compare);
14639 %}
14640
14641 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14642 %{
14643 match(Set cr (CmpF src1 src2));
14644
14645 ins_cost(3 * INSN_COST);
14646 format %{ "fcmps $src1, 0.0" %}
14647
14648 ins_encode %{
14649 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14650 %}
14651
14652 ins_pipe(pipe_class_compare);
14653 %}
14654 // FROM HERE
14655
14656 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14657 %{
14658 match(Set cr (CmpD src1 src2));
14659
14660 ins_cost(3 * INSN_COST);
14661 format %{ "fcmpd $src1, $src2" %}
14662
14663 ins_encode %{
14664 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14665 %}
14666
14667 ins_pipe(pipe_class_compare);
14668 %}
14669
14670 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14671 %{
14672 match(Set cr (CmpD src1 src2));
14673
14674 ins_cost(3 * INSN_COST);
14675 format %{ "fcmpd $src1, 0.0" %}
14676
14677 ins_encode %{
14678 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14679 %}
14680
14681 ins_pipe(pipe_class_compare);
14682 %}
14683
14684 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14685 %{
14686 match(Set dst (CmpF3 src1 src2));
14687 effect(KILL cr);
14688
14689 ins_cost(5 * INSN_COST);
14690 format %{ "fcmps $src1, $src2\n\t"
14691 "csinvw($dst, zr, zr, eq\n\t"
14692 "csnegw($dst, $dst, $dst, lt)"
14693 %}
14694
14695 ins_encode %{
14696 Label done;
14697 FloatRegister s1 = as_FloatRegister($src1$$reg);
14698 FloatRegister s2 = as_FloatRegister($src2$$reg);
14699 Register d = as_Register($dst$$reg);
14700 __ fcmps(s1, s2);
14701 // installs 0 if EQ else -1
14702 __ csinvw(d, zr, zr, Assembler::EQ);
14703 // keeps -1 if less or unordered else installs 1
14704 __ csnegw(d, d, d, Assembler::LT);
14705 __ bind(done);
14706 %}
14707
14708 ins_pipe(pipe_class_default);
14709
14710 %}
14711
14712 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14713 %{
14714 match(Set dst (CmpD3 src1 src2));
14715 effect(KILL cr);
14716
14717 ins_cost(5 * INSN_COST);
14718 format %{ "fcmpd $src1, $src2\n\t"
14719 "csinvw($dst, zr, zr, eq\n\t"
14720 "csnegw($dst, $dst, $dst, lt)"
14721 %}
14722
14723 ins_encode %{
14724 Label done;
14725 FloatRegister s1 = as_FloatRegister($src1$$reg);
14726 FloatRegister s2 = as_FloatRegister($src2$$reg);
14727 Register d = as_Register($dst$$reg);
14728 __ fcmpd(s1, s2);
14729 // installs 0 if EQ else -1
14730 __ csinvw(d, zr, zr, Assembler::EQ);
14731 // keeps -1 if less or unordered else installs 1
14732 __ csnegw(d, d, d, Assembler::LT);
14733 __ bind(done);
14734 %}
14735 ins_pipe(pipe_class_default);
14736
14737 %}
14738
14739 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14740 %{
14741 match(Set dst (CmpF3 src1 zero));
14742 effect(KILL cr);
14743
14744 ins_cost(5 * INSN_COST);
14745 format %{ "fcmps $src1, 0.0\n\t"
14746 "csinvw($dst, zr, zr, eq\n\t"
14747 "csnegw($dst, $dst, $dst, lt)"
14748 %}
14749
14750 ins_encode %{
14751 Label done;
14752 FloatRegister s1 = as_FloatRegister($src1$$reg);
14753 Register d = as_Register($dst$$reg);
14754 __ fcmps(s1, 0.0);
14755 // installs 0 if EQ else -1
14756 __ csinvw(d, zr, zr, Assembler::EQ);
14757 // keeps -1 if less or unordered else installs 1
14758 __ csnegw(d, d, d, Assembler::LT);
14759 __ bind(done);
14760 %}
14761
14762 ins_pipe(pipe_class_default);
14763
14764 %}
14765
14766 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14767 %{
14768 match(Set dst (CmpD3 src1 zero));
14769 effect(KILL cr);
14770
14771 ins_cost(5 * INSN_COST);
14772 format %{ "fcmpd $src1, 0.0\n\t"
14773 "csinvw($dst, zr, zr, eq\n\t"
14774 "csnegw($dst, $dst, $dst, lt)"
14775 %}
14776
14777 ins_encode %{
14778 Label done;
14779 FloatRegister s1 = as_FloatRegister($src1$$reg);
14780 Register d = as_Register($dst$$reg);
14781 __ fcmpd(s1, 0.0);
14782 // installs 0 if EQ else -1
14783 __ csinvw(d, zr, zr, Assembler::EQ);
14784 // keeps -1 if less or unordered else installs 1
14785 __ csnegw(d, d, d, Assembler::LT);
14786 __ bind(done);
14787 %}
14788 ins_pipe(pipe_class_default);
14789
14790 %}
14791
14792 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14793 %{
14794 match(Set dst (CmpLTMask p q));
14795 effect(KILL cr);
14796
14797 ins_cost(3 * INSN_COST);
14798
14799 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14800 "csetw $dst, lt\n\t"
14801 "subw $dst, zr, $dst"
14802 %}
14803
14804 ins_encode %{
14805 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14806 __ csetw(as_Register($dst$$reg), Assembler::LT);
14807 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14808 %}
14809
14810 ins_pipe(ialu_reg_reg);
14811 %}
14812
14813 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14814 %{
14815 match(Set dst (CmpLTMask src zero));
14816 effect(KILL cr);
14817
14818 ins_cost(INSN_COST);
14819
14820 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14821
14822 ins_encode %{
14823 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14824 %}
14825
14826 ins_pipe(ialu_reg_shift);
14827 %}
14828
14829 // ============================================================================
14830 // Max and Min
14831
14832 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14833
14834 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14835 %{
14836 effect(DEF cr, USE src);
14837 ins_cost(INSN_COST);
14838 format %{ "cmpw $src, 0" %}
14839
14840 ins_encode %{
14841 __ cmpw($src$$Register, 0);
14842 %}
14843 ins_pipe(icmp_reg_imm);
14844 %}
14845
14846 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14847 %{
14848 match(Set dst (MinI src1 src2));
14849 ins_cost(INSN_COST * 3);
14850
14851 expand %{
14852 rFlagsReg cr;
14853 compI_reg_reg(cr, src1, src2);
14854 cmovI_reg_reg_lt(dst, src1, src2, cr);
14855 %}
14856 %}
14857
14858 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14859 %{
14860 match(Set dst (MaxI src1 src2));
14861 ins_cost(INSN_COST * 3);
14862
14863 expand %{
14864 rFlagsReg cr;
14865 compI_reg_reg(cr, src1, src2);
14866 cmovI_reg_reg_gt(dst, src1, src2, cr);
14867 %}
14868 %}
14869
14870
14871 // ============================================================================
14872 // Branch Instructions
14873
14874 // Direct Branch.
14875 instruct branch(label lbl)
14876 %{
14877 match(Goto);
14878
14879 effect(USE lbl);
14880
14881 ins_cost(BRANCH_COST);
14882 format %{ "b $lbl" %}
14883
14884 ins_encode(aarch64_enc_b(lbl));
14885
14886 ins_pipe(pipe_branch);
14887 %}
14888
14889 // Conditional Near Branch
14890 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14891 %{
14892 // Same match rule as `branchConFar'.
14893 match(If cmp cr);
14894
14895 effect(USE lbl);
14896
14897 ins_cost(BRANCH_COST);
14898 // If set to 1 this indicates that the current instruction is a
14899 // short variant of a long branch. This avoids using this
14900 // instruction in first-pass matching. It will then only be used in
14901 // the `Shorten_branches' pass.
14902 // ins_short_branch(1);
14903 format %{ "b$cmp $lbl" %}
14904
14905 ins_encode(aarch64_enc_br_con(cmp, lbl));
14906
14907 ins_pipe(pipe_branch_cond);
14908 %}
14909
14910 // Conditional Near Branch Unsigned
14911 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14912 %{
14913 // Same match rule as `branchConFar'.
14914 match(If cmp cr);
14915
14916 effect(USE lbl);
14917
14918 ins_cost(BRANCH_COST);
14919 // If set to 1 this indicates that the current instruction is a
14920 // short variant of a long branch. This avoids using this
14921 // instruction in first-pass matching. It will then only be used in
14922 // the `Shorten_branches' pass.
14923 // ins_short_branch(1);
14924 format %{ "b$cmp $lbl\t# unsigned" %}
14925
14926 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14927
14928 ins_pipe(pipe_branch_cond);
14929 %}
14930
14931 // Make use of CBZ and CBNZ. These instructions, as well as being
14932 // shorter than (cmp; branch), have the additional benefit of not
14933 // killing the flags.
14934
14935 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14936 match(If cmp (CmpI op1 op2));
14937 effect(USE labl);
14938
14939 ins_cost(BRANCH_COST);
14940 format %{ "cbw$cmp $op1, $labl" %}
14941 ins_encode %{
14942 Label* L = $labl$$label;
14943 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14944 if (cond == Assembler::EQ)
14945 __ cbzw($op1$$Register, *L);
14946 else
14947 __ cbnzw($op1$$Register, *L);
14948 %}
14949 ins_pipe(pipe_cmp_branch);
14950 %}
14951
14952 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14953 match(If cmp (CmpL op1 op2));
14954 effect(USE labl);
14955
14956 ins_cost(BRANCH_COST);
14957 format %{ "cb$cmp $op1, $labl" %}
14958 ins_encode %{
14959 Label* L = $labl$$label;
14960 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14961 if (cond == Assembler::EQ)
14962 __ cbz($op1$$Register, *L);
14963 else
14964 __ cbnz($op1$$Register, *L);
14965 %}
14966 ins_pipe(pipe_cmp_branch);
14967 %}
14968
14969 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14970 match(If cmp (CmpP op1 op2));
14971 effect(USE labl);
14972
14973 ins_cost(BRANCH_COST);
14974 format %{ "cb$cmp $op1, $labl" %}
14975 ins_encode %{
14976 Label* L = $labl$$label;
14977 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14978 if (cond == Assembler::EQ)
14979 __ cbz($op1$$Register, *L);
14980 else
14981 __ cbnz($op1$$Register, *L);
14982 %}
14983 ins_pipe(pipe_cmp_branch);
14984 %}
14985
14986 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14987 match(If cmp (CmpN op1 op2));
14988 effect(USE labl);
14989
14990 ins_cost(BRANCH_COST);
14991 format %{ "cbw$cmp $op1, $labl" %}
14992 ins_encode %{
14993 Label* L = $labl$$label;
14994 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14995 if (cond == Assembler::EQ)
14996 __ cbzw($op1$$Register, *L);
14997 else
14998 __ cbnzw($op1$$Register, *L);
14999 %}
15000 ins_pipe(pipe_cmp_branch);
15001 %}
15002
15003 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15004 match(If cmp (CmpP (DecodeN oop) zero));
15005 effect(USE labl);
15006
15007 ins_cost(BRANCH_COST);
15008 format %{ "cb$cmp $oop, $labl" %}
15009 ins_encode %{
15010 Label* L = $labl$$label;
15011 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15012 if (cond == Assembler::EQ)
15013 __ cbzw($oop$$Register, *L);
15014 else
15015 __ cbnzw($oop$$Register, *L);
15016 %}
15017 ins_pipe(pipe_cmp_branch);
15018 %}
15019
15020 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15021 match(If cmp (CmpU op1 op2));
15022 effect(USE labl);
15023
15024 ins_cost(BRANCH_COST);
15025 format %{ "cbw$cmp $op1, $labl" %}
15026 ins_encode %{
15027 Label* L = $labl$$label;
15028 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15029 if (cond == Assembler::EQ || cond == Assembler::LS) {
15030 __ cbzw($op1$$Register, *L);
15031 } else {
15032 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15033 __ cbnzw($op1$$Register, *L);
15034 }
15035 %}
15036 ins_pipe(pipe_cmp_branch);
15037 %}
15038
15039 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15040 match(If cmp (CmpUL op1 op2));
15041 effect(USE labl);
15042
15043 ins_cost(BRANCH_COST);
15044 format %{ "cb$cmp $op1, $labl" %}
15045 ins_encode %{
15046 Label* L = $labl$$label;
15047 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15048 if (cond == Assembler::EQ || cond == Assembler::LS) {
15049 __ cbz($op1$$Register, *L);
15050 } else {
15051 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15052 __ cbnz($op1$$Register, *L);
15053 }
15054 %}
15055 ins_pipe(pipe_cmp_branch);
15056 %}
15057
15058 // Test bit and Branch
15059
15060 // Patterns for short (< 32KiB) variants
15061 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15062 match(If cmp (CmpL op1 op2));
15063 effect(USE labl);
15064
15065 ins_cost(BRANCH_COST);
15066 format %{ "cb$cmp $op1, $labl # long" %}
15067 ins_encode %{
15068 Label* L = $labl$$label;
15069 Assembler::Condition cond =
15070 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15071 __ tbr(cond, $op1$$Register, 63, *L);
15072 %}
15073 ins_pipe(pipe_cmp_branch);
15074 ins_short_branch(1);
15075 %}
15076
15077 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15078 match(If cmp (CmpI op1 op2));
15079 effect(USE labl);
15080
15081 ins_cost(BRANCH_COST);
15082 format %{ "cb$cmp $op1, $labl # int" %}
15083 ins_encode %{
15084 Label* L = $labl$$label;
15085 Assembler::Condition cond =
15086 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15087 __ tbr(cond, $op1$$Register, 31, *L);
15088 %}
15089 ins_pipe(pipe_cmp_branch);
15090 ins_short_branch(1);
15091 %}
15092
15093 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15094 match(If cmp (CmpL (AndL op1 op2) op3));
15095 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15096 effect(USE labl);
15097
15098 ins_cost(BRANCH_COST);
15099 format %{ "tb$cmp $op1, $op2, $labl" %}
15100 ins_encode %{
15101 Label* L = $labl$$label;
15102 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15103 int bit = exact_log2_long($op2$$constant);
15104 __ tbr(cond, $op1$$Register, bit, *L);
15105 %}
15106 ins_pipe(pipe_cmp_branch);
15107 ins_short_branch(1);
15108 %}
15109
15110 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15111 match(If cmp (CmpI (AndI op1 op2) op3));
15112 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15113 effect(USE labl);
15114
15115 ins_cost(BRANCH_COST);
15116 format %{ "tb$cmp $op1, $op2, $labl" %}
15117 ins_encode %{
15118 Label* L = $labl$$label;
15119 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15120 int bit = exact_log2((juint)$op2$$constant);
15121 __ tbr(cond, $op1$$Register, bit, *L);
15122 %}
15123 ins_pipe(pipe_cmp_branch);
15124 ins_short_branch(1);
15125 %}
15126
15127 // And far variants
15128 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15129 match(If cmp (CmpL op1 op2));
15130 effect(USE labl);
15131
15132 ins_cost(BRANCH_COST);
15133 format %{ "cb$cmp $op1, $labl # long" %}
15134 ins_encode %{
15135 Label* L = $labl$$label;
15136 Assembler::Condition cond =
15137 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15138 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15139 %}
15140 ins_pipe(pipe_cmp_branch);
15141 %}
15142
15143 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15144 match(If cmp (CmpI op1 op2));
15145 effect(USE labl);
15146
15147 ins_cost(BRANCH_COST);
15148 format %{ "cb$cmp $op1, $labl # int" %}
15149 ins_encode %{
15150 Label* L = $labl$$label;
15151 Assembler::Condition cond =
15152 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15153 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15154 %}
15155 ins_pipe(pipe_cmp_branch);
15156 %}
15157
15158 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15159 match(If cmp (CmpL (AndL op1 op2) op3));
15160 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15161 effect(USE labl);
15162
15163 ins_cost(BRANCH_COST);
15164 format %{ "tb$cmp $op1, $op2, $labl" %}
15165 ins_encode %{
15166 Label* L = $labl$$label;
15167 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15168 int bit = exact_log2_long($op2$$constant);
15169 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15170 %}
15171 ins_pipe(pipe_cmp_branch);
15172 %}
15173
15174 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15175 match(If cmp (CmpI (AndI op1 op2) op3));
15176 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15177 effect(USE labl);
15178
15179 ins_cost(BRANCH_COST);
15180 format %{ "tb$cmp $op1, $op2, $labl" %}
15181 ins_encode %{
15182 Label* L = $labl$$label;
15183 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15184 int bit = exact_log2((juint)$op2$$constant);
15185 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15186 %}
15187 ins_pipe(pipe_cmp_branch);
15188 %}
15189
15190 // Test bits
15191
15192 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15193 match(Set cr (CmpL (AndL op1 op2) op3));
15194 predicate(Assembler::operand_valid_for_logical_immediate
15195 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15196
15197 ins_cost(INSN_COST);
15198 format %{ "tst $op1, $op2 # long" %}
15199 ins_encode %{
15200 __ tst($op1$$Register, $op2$$constant);
15201 %}
15202 ins_pipe(ialu_reg_reg);
15203 %}
15204
15205 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15206 match(Set cr (CmpI (AndI op1 op2) op3));
15207 predicate(Assembler::operand_valid_for_logical_immediate
15208 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15209
15210 ins_cost(INSN_COST);
15211 format %{ "tst $op1, $op2 # int" %}
15212 ins_encode %{
15213 __ tstw($op1$$Register, $op2$$constant);
15214 %}
15215 ins_pipe(ialu_reg_reg);
15216 %}
15217
15218 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15219 match(Set cr (CmpL (AndL op1 op2) op3));
15220
15221 ins_cost(INSN_COST);
15222 format %{ "tst $op1, $op2 # long" %}
15223 ins_encode %{
15224 __ tst($op1$$Register, $op2$$Register);
15225 %}
15226 ins_pipe(ialu_reg_reg);
15227 %}
15228
15229 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15230 match(Set cr (CmpI (AndI op1 op2) op3));
15231
15232 ins_cost(INSN_COST);
15233 format %{ "tstw $op1, $op2 # int" %}
15234 ins_encode %{
15235 __ tstw($op1$$Register, $op2$$Register);
15236 %}
15237 ins_pipe(ialu_reg_reg);
15238 %}
15239
15240
15241 // Conditional Far Branch
15242 // Conditional Far Branch Unsigned
15243 // TODO: fixme
15244
15245 // counted loop end branch near
15246 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15247 %{
15248 match(CountedLoopEnd cmp cr);
15249
15250 effect(USE lbl);
15251
15252 ins_cost(BRANCH_COST);
15253 // short variant.
15254 // ins_short_branch(1);
15255 format %{ "b$cmp $lbl \t// counted loop end" %}
15256
15257 ins_encode(aarch64_enc_br_con(cmp, lbl));
15258
15259 ins_pipe(pipe_branch);
15260 %}
15261
15262 // counted loop end branch far
15263 // TODO: fixme
15264
15265 // ============================================================================
15266 // inlined locking and unlocking
15267
15268 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15269 %{
15270 match(Set cr (FastLock object box));
15271 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15272
15273 ins_cost(5 * INSN_COST);
15274 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15275
15276 ins_encode %{
15277 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15278 %}
15279
15280 ins_pipe(pipe_serial);
15281 %}
15282
15283 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15284 %{
15285 match(Set cr (FastUnlock object box));
15286 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15287
15288 ins_cost(5 * INSN_COST);
15289 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15290
15291 ins_encode %{
15292 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15293 %}
15294
15295 ins_pipe(pipe_serial);
15296 %}
15297
15298 // ============================================================================
15299 // Safepoint Instructions
15300
15301 // TODO
15302 // provide a near and far version of this code
15303
15304 instruct safePoint(rFlagsReg cr, iRegP poll)
15305 %{
15306 match(SafePoint poll);
15307 effect(KILL cr);
15308
15309 format %{
15310 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15311 %}
15312 ins_encode %{
15313 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15314 %}
15315 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15316 %}
15317
15318
15319 // ============================================================================
15320 // Procedure Call/Return Instructions
15321
15322 // Call Java Static Instruction
15323
15324 instruct CallStaticJavaDirect(method meth)
15325 %{
15326 match(CallStaticJava);
15327
15328 effect(USE meth);
15329
15330 ins_cost(CALL_COST);
15331
15332 format %{ "call,static $meth \t// ==> " %}
15333
15334 ins_encode(aarch64_enc_java_static_call(meth),
15335 aarch64_enc_call_epilog);
15336
15337 ins_pipe(pipe_class_call);
15338 %}
15339
15340 // TO HERE
15341
15342 // Call Java Dynamic Instruction
15343 instruct CallDynamicJavaDirect(method meth)
15344 %{
15345 match(CallDynamicJava);
15346
15347 effect(USE meth);
15348
15349 ins_cost(CALL_COST);
15350
15351 format %{ "CALL,dynamic $meth \t// ==> " %}
15352
15353 ins_encode(aarch64_enc_java_dynamic_call(meth),
15354 aarch64_enc_call_epilog);
15355
15356 ins_pipe(pipe_class_call);
15357 %}
15358
15359 // Call Runtime Instruction
15360
15361 instruct CallRuntimeDirect(method meth)
15362 %{
15363 match(CallRuntime);
15364
15365 effect(USE meth);
15366
15367 ins_cost(CALL_COST);
15368
15369 format %{ "CALL, runtime $meth" %}
15370
15371 ins_encode( aarch64_enc_java_to_runtime(meth) );
15372
15373 ins_pipe(pipe_class_call);
15374 %}
15375
15376 // Call Runtime Instruction
15377
15378 instruct CallLeafDirect(method meth)
15379 %{
15380 match(CallLeaf);
15381
15382 effect(USE meth);
15383
15384 ins_cost(CALL_COST);
15385
15386 format %{ "CALL, runtime leaf $meth" %}
15387
15388 ins_encode( aarch64_enc_java_to_runtime(meth) );
15389
15390 ins_pipe(pipe_class_call);
15391 %}
15392
15393 // Call Runtime Instruction without safepoint and with vector arguments
15394 instruct CallLeafDirectVector(method meth)
15395 %{
15396 match(CallLeafVector);
15397
15398 effect(USE meth);
15399
15400 ins_cost(CALL_COST);
15401
15402 format %{ "CALL, runtime leaf vector $meth" %}
15403
15404 ins_encode(aarch64_enc_java_to_runtime(meth));
15405
15406 ins_pipe(pipe_class_call);
15407 %}
15408
15409 // Call Runtime Instruction
15410
15411 instruct CallLeafNoFPDirect(method meth)
15412 %{
15413 match(CallLeafNoFP);
15414
15415 effect(USE meth);
15416
15417 ins_cost(CALL_COST);
15418
15419 format %{ "CALL, runtime leaf nofp $meth" %}
15420
15421 ins_encode( aarch64_enc_java_to_runtime(meth) );
15422
15423 ins_pipe(pipe_class_call);
15424 %}
15425
15426 // Tail Call; Jump from runtime stub to Java code.
15427 // Also known as an 'interprocedural jump'.
15428 // Target of jump will eventually return to caller.
15429 // TailJump below removes the return address.
15430 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15431 // emitted just above the TailCall which has reset rfp to the caller state.
15432 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15433 %{
15434 match(TailCall jump_target method_ptr);
15435
15436 ins_cost(CALL_COST);
15437
15438 format %{ "br $jump_target\t# $method_ptr holds method" %}
15439
15440 ins_encode(aarch64_enc_tail_call(jump_target));
15441
15442 ins_pipe(pipe_class_call);
15443 %}
15444
15445 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15446 %{
15447 match(TailJump jump_target ex_oop);
15448
15449 ins_cost(CALL_COST);
15450
15451 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15452
15453 ins_encode(aarch64_enc_tail_jmp(jump_target));
15454
15455 ins_pipe(pipe_class_call);
15456 %}
15457
15458 // Forward exception.
15459 instruct ForwardExceptionjmp()
15460 %{
15461 match(ForwardException);
15462 ins_cost(CALL_COST);
15463
15464 format %{ "b forward_exception_stub" %}
15465 ins_encode %{
15466 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15467 %}
15468 ins_pipe(pipe_class_call);
15469 %}
15470
15471 // Create exception oop: created by stack-crawling runtime code.
15472 // Created exception is now available to this handler, and is setup
15473 // just prior to jumping to this handler. No code emitted.
15474 // TODO check
15475 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15476 instruct CreateException(iRegP_R0 ex_oop)
15477 %{
15478 match(Set ex_oop (CreateEx));
15479
15480 format %{ " -- \t// exception oop; no code emitted" %}
15481
15482 size(0);
15483
15484 ins_encode( /*empty*/ );
15485
15486 ins_pipe(pipe_class_empty);
15487 %}
15488
15489 // Rethrow exception: The exception oop will come in the first
15490 // argument position. Then JUMP (not call) to the rethrow stub code.
15491 instruct RethrowException() %{
15492 match(Rethrow);
15493 ins_cost(CALL_COST);
15494
15495 format %{ "b rethrow_stub" %}
15496
15497 ins_encode( aarch64_enc_rethrow() );
15498
15499 ins_pipe(pipe_class_call);
15500 %}
15501
15502
15503 // Return Instruction
15504 // epilog node loads ret address into lr as part of frame pop
15505 instruct Ret()
15506 %{
15507 match(Return);
15508
15509 format %{ "ret\t// return register" %}
15510
15511 ins_encode( aarch64_enc_ret() );
15512
15513 ins_pipe(pipe_branch);
15514 %}
15515
15516 // Die now.
15517 instruct ShouldNotReachHere() %{
15518 match(Halt);
15519
15520 ins_cost(CALL_COST);
15521 format %{ "ShouldNotReachHere" %}
15522
15523 ins_encode %{
15524 if (is_reachable()) {
15525 const char* str = __ code_string(_halt_reason);
15526 __ stop(str);
15527 }
15528 %}
15529
15530 ins_pipe(pipe_class_default);
15531 %}
15532
15533 // ============================================================================
15534 // Partial Subtype Check
15535 //
15536 // superklass array for an instance of the superklass. Set a hidden
15537 // internal cache on a hit (cache is checked with exposed code in
15538 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15539 // encoding ALSO sets flags.
15540
15541 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15542 %{
15543 match(Set result (PartialSubtypeCheck sub super));
15544 predicate(!UseSecondarySupersTable);
15545 effect(KILL cr, KILL temp);
15546
15547 ins_cost(20 * INSN_COST); // slightly larger than the next version
15548 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15549
15550 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15551
15552 opcode(0x1); // Force zero of result reg on hit
15553
15554 ins_pipe(pipe_class_memory);
15555 %}
15556
15557 // Two versions of partialSubtypeCheck, both used when we need to
15558 // search for a super class in the secondary supers array. The first
15559 // is used when we don't know _a priori_ the class being searched
15560 // for. The second, far more common, is used when we do know: this is
15561 // used for instanceof, checkcast, and any case where C2 can determine
15562 // it by constant propagation.
15563
15564 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15565 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15566 rFlagsReg cr)
15567 %{
15568 match(Set result (PartialSubtypeCheck sub super));
15569 predicate(UseSecondarySupersTable);
15570 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15571
15572 ins_cost(10 * INSN_COST); // slightly larger than the next version
15573 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15574
15575 ins_encode %{
15576 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15577 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15578 $vtemp$$FloatRegister,
15579 $result$$Register, /*L_success*/nullptr);
15580 %}
15581
15582 ins_pipe(pipe_class_memory);
15583 %}
15584
15585 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15586 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15587 rFlagsReg cr)
15588 %{
15589 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15590 predicate(UseSecondarySupersTable);
15591 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15592
15593 ins_cost(5 * INSN_COST); // smaller than the next version
15594 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15595
15596 ins_encode %{
15597 bool success = false;
15598 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15599 if (InlineSecondarySupersTest) {
15600 success =
15601 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15602 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15603 $vtemp$$FloatRegister,
15604 $result$$Register,
15605 super_klass_slot);
15606 } else {
15607 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15608 success = (call != nullptr);
15609 }
15610 if (!success) {
15611 ciEnv::current()->record_failure("CodeCache is full");
15612 return;
15613 }
15614 %}
15615
15616 ins_pipe(pipe_class_memory);
15617 %}
15618
15619 // Intrisics for String.compareTo()
15620
15621 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15622 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15623 %{
15624 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15625 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15626 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15627
15628 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15629 ins_encode %{
15630 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15631 __ string_compare($str1$$Register, $str2$$Register,
15632 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15633 $tmp1$$Register, $tmp2$$Register,
15634 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15635 %}
15636 ins_pipe(pipe_class_memory);
15637 %}
15638
15639 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15640 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15641 %{
15642 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15643 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15644 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15645
15646 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15647 ins_encode %{
15648 __ string_compare($str1$$Register, $str2$$Register,
15649 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15650 $tmp1$$Register, $tmp2$$Register,
15651 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15652 %}
15653 ins_pipe(pipe_class_memory);
15654 %}
15655
15656 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15657 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15658 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15659 %{
15660 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15661 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15662 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15663 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15664
15665 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15666 ins_encode %{
15667 __ string_compare($str1$$Register, $str2$$Register,
15668 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15669 $tmp1$$Register, $tmp2$$Register,
15670 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15671 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15672 %}
15673 ins_pipe(pipe_class_memory);
15674 %}
15675
15676 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15677 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15678 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15679 %{
15680 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15681 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15682 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15683 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15684
15685 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15686 ins_encode %{
15687 __ string_compare($str1$$Register, $str2$$Register,
15688 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15689 $tmp1$$Register, $tmp2$$Register,
15690 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15691 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15692 %}
15693 ins_pipe(pipe_class_memory);
15694 %}
15695
15696 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15697 // these string_compare variants as NEON register type for convenience so that the prototype of
15698 // string_compare can be shared with all variants.
15699
15700 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15701 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15702 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15703 pRegGov_P1 pgtmp2, rFlagsReg cr)
15704 %{
15705 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15706 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15707 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15708 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15709
15710 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15711 ins_encode %{
15712 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15713 __ string_compare($str1$$Register, $str2$$Register,
15714 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15715 $tmp1$$Register, $tmp2$$Register,
15716 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15717 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15718 StrIntrinsicNode::LL);
15719 %}
15720 ins_pipe(pipe_class_memory);
15721 %}
15722
15723 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15724 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15725 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15726 pRegGov_P1 pgtmp2, rFlagsReg cr)
15727 %{
15728 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15729 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15730 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15731 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15732
15733 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15734 ins_encode %{
15735 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15736 __ string_compare($str1$$Register, $str2$$Register,
15737 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15738 $tmp1$$Register, $tmp2$$Register,
15739 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15740 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15741 StrIntrinsicNode::LU);
15742 %}
15743 ins_pipe(pipe_class_memory);
15744 %}
15745
15746 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15747 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15748 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15749 pRegGov_P1 pgtmp2, rFlagsReg cr)
15750 %{
15751 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15752 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15753 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15754 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15755
15756 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15757 ins_encode %{
15758 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15759 __ string_compare($str1$$Register, $str2$$Register,
15760 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15761 $tmp1$$Register, $tmp2$$Register,
15762 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15763 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15764 StrIntrinsicNode::UL);
15765 %}
15766 ins_pipe(pipe_class_memory);
15767 %}
15768
15769 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15770 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15771 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15772 pRegGov_P1 pgtmp2, rFlagsReg cr)
15773 %{
15774 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15775 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15776 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15777 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15778
15779 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15780 ins_encode %{
15781 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15782 __ string_compare($str1$$Register, $str2$$Register,
15783 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15784 $tmp1$$Register, $tmp2$$Register,
15785 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15786 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15787 StrIntrinsicNode::UU);
15788 %}
15789 ins_pipe(pipe_class_memory);
15790 %}
15791
15792 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15793 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15794 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15795 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15796 %{
15797 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15798 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15799 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15800 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15801 TEMP vtmp0, TEMP vtmp1, KILL cr);
15802 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15803 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15804
15805 ins_encode %{
15806 __ string_indexof($str1$$Register, $str2$$Register,
15807 $cnt1$$Register, $cnt2$$Register,
15808 $tmp1$$Register, $tmp2$$Register,
15809 $tmp3$$Register, $tmp4$$Register,
15810 $tmp5$$Register, $tmp6$$Register,
15811 -1, $result$$Register, StrIntrinsicNode::UU);
15812 %}
15813 ins_pipe(pipe_class_memory);
15814 %}
15815
15816 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15817 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15818 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15819 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15820 %{
15821 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15822 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15823 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15824 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15825 TEMP vtmp0, TEMP vtmp1, KILL cr);
15826 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15827 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15828
15829 ins_encode %{
15830 __ string_indexof($str1$$Register, $str2$$Register,
15831 $cnt1$$Register, $cnt2$$Register,
15832 $tmp1$$Register, $tmp2$$Register,
15833 $tmp3$$Register, $tmp4$$Register,
15834 $tmp5$$Register, $tmp6$$Register,
15835 -1, $result$$Register, StrIntrinsicNode::LL);
15836 %}
15837 ins_pipe(pipe_class_memory);
15838 %}
15839
15840 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15841 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15842 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15843 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15844 %{
15845 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15846 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15847 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15848 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15849 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15850 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15851 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15852
15853 ins_encode %{
15854 __ string_indexof($str1$$Register, $str2$$Register,
15855 $cnt1$$Register, $cnt2$$Register,
15856 $tmp1$$Register, $tmp2$$Register,
15857 $tmp3$$Register, $tmp4$$Register,
15858 $tmp5$$Register, $tmp6$$Register,
15859 -1, $result$$Register, StrIntrinsicNode::UL);
15860 %}
15861 ins_pipe(pipe_class_memory);
15862 %}
15863
15864 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15865 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15866 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15867 %{
15868 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15869 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15870 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15871 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15872 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15873 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15874
15875 ins_encode %{
15876 int icnt2 = (int)$int_cnt2$$constant;
15877 __ string_indexof($str1$$Register, $str2$$Register,
15878 $cnt1$$Register, zr,
15879 $tmp1$$Register, $tmp2$$Register,
15880 $tmp3$$Register, $tmp4$$Register, zr, zr,
15881 icnt2, $result$$Register, StrIntrinsicNode::UU);
15882 %}
15883 ins_pipe(pipe_class_memory);
15884 %}
15885
15886 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15887 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15888 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15889 %{
15890 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15891 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15892 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15893 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15894 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15895 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15896
15897 ins_encode %{
15898 int icnt2 = (int)$int_cnt2$$constant;
15899 __ string_indexof($str1$$Register, $str2$$Register,
15900 $cnt1$$Register, zr,
15901 $tmp1$$Register, $tmp2$$Register,
15902 $tmp3$$Register, $tmp4$$Register, zr, zr,
15903 icnt2, $result$$Register, StrIntrinsicNode::LL);
15904 %}
15905 ins_pipe(pipe_class_memory);
15906 %}
15907
15908 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15909 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15910 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15911 %{
15912 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15913 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15914 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15915 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15916 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15917 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15918
15919 ins_encode %{
15920 int icnt2 = (int)$int_cnt2$$constant;
15921 __ string_indexof($str1$$Register, $str2$$Register,
15922 $cnt1$$Register, zr,
15923 $tmp1$$Register, $tmp2$$Register,
15924 $tmp3$$Register, $tmp4$$Register, zr, zr,
15925 icnt2, $result$$Register, StrIntrinsicNode::UL);
15926 %}
15927 ins_pipe(pipe_class_memory);
15928 %}
15929
15930 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15931 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15932 iRegINoSp tmp3, rFlagsReg cr)
15933 %{
15934 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15935 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15936 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15937 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15938
15939 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15940
15941 ins_encode %{
15942 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15943 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15944 $tmp3$$Register);
15945 %}
15946 ins_pipe(pipe_class_memory);
15947 %}
15948
15949 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15950 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15951 iRegINoSp tmp3, rFlagsReg cr)
15952 %{
15953 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15954 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15955 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15956 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15957
15958 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15959
15960 ins_encode %{
15961 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15962 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15963 $tmp3$$Register);
15964 %}
15965 ins_pipe(pipe_class_memory);
15966 %}
15967
15968 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15969 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15970 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15971 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15972 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15973 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15974 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15975 ins_encode %{
15976 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15977 $result$$Register, $ztmp1$$FloatRegister,
15978 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15979 $ptmp$$PRegister, true /* isL */);
15980 %}
15981 ins_pipe(pipe_class_memory);
15982 %}
15983
15984 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15985 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15986 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15987 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15988 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15989 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15990 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15991 ins_encode %{
15992 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15993 $result$$Register, $ztmp1$$FloatRegister,
15994 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15995 $ptmp$$PRegister, false /* isL */);
15996 %}
15997 ins_pipe(pipe_class_memory);
15998 %}
15999
16000 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16001 iRegI_R0 result, rFlagsReg cr)
16002 %{
16003 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16004 match(Set result (StrEquals (Binary str1 str2) cnt));
16005 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16006
16007 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16008 ins_encode %{
16009 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16010 __ string_equals($str1$$Register, $str2$$Register,
16011 $result$$Register, $cnt$$Register);
16012 %}
16013 ins_pipe(pipe_class_memory);
16014 %}
16015
16016 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16017 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16018 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16019 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16020 iRegP_R10 tmp, rFlagsReg cr)
16021 %{
16022 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16023 match(Set result (AryEq ary1 ary2));
16024 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16025 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16026 TEMP vtmp6, TEMP vtmp7, KILL cr);
16027
16028 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16029 ins_encode %{
16030 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16031 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16032 $result$$Register, $tmp$$Register, 1);
16033 if (tpc == nullptr) {
16034 ciEnv::current()->record_failure("CodeCache is full");
16035 return;
16036 }
16037 %}
16038 ins_pipe(pipe_class_memory);
16039 %}
16040
16041 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16042 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16043 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16044 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16045 iRegP_R10 tmp, rFlagsReg cr)
16046 %{
16047 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16048 match(Set result (AryEq ary1 ary2));
16049 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16050 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16051 TEMP vtmp6, TEMP vtmp7, KILL cr);
16052
16053 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16054 ins_encode %{
16055 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16056 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16057 $result$$Register, $tmp$$Register, 2);
16058 if (tpc == nullptr) {
16059 ciEnv::current()->record_failure("CodeCache is full");
16060 return;
16061 }
16062 %}
16063 ins_pipe(pipe_class_memory);
16064 %}
16065
16066 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16067 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16068 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16069 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16070 %{
16071 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16072 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16073 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16074
16075 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16076 ins_encode %{
16077 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16078 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16079 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16080 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16081 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16082 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16083 (BasicType)$basic_type$$constant);
16084 if (tpc == nullptr) {
16085 ciEnv::current()->record_failure("CodeCache is full");
16086 return;
16087 }
16088 %}
16089 ins_pipe(pipe_class_memory);
16090 %}
16091
16092 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16093 %{
16094 match(Set result (CountPositives ary1 len));
16095 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16096 format %{ "count positives byte[] $ary1,$len -> $result" %}
16097 ins_encode %{
16098 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16099 if (tpc == nullptr) {
16100 ciEnv::current()->record_failure("CodeCache is full");
16101 return;
16102 }
16103 %}
16104 ins_pipe( pipe_slow );
16105 %}
16106
16107 // fast char[] to byte[] compression
16108 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16109 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16110 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16111 iRegI_R0 result, rFlagsReg cr)
16112 %{
16113 match(Set result (StrCompressedCopy src (Binary dst len)));
16114 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16115 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16116
16117 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16118 ins_encode %{
16119 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16120 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16121 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16122 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16123 %}
16124 ins_pipe(pipe_slow);
16125 %}
16126
16127 // fast byte[] to char[] inflation
16128 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16129 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16130 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16131 %{
16132 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16133 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16134 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16135 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16136
16137 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16138 ins_encode %{
16139 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16140 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16141 $vtmp2$$FloatRegister, $tmp$$Register);
16142 if (tpc == nullptr) {
16143 ciEnv::current()->record_failure("CodeCache is full");
16144 return;
16145 }
16146 %}
16147 ins_pipe(pipe_class_memory);
16148 %}
16149
16150 // encode char[] to byte[] in ISO_8859_1
16151 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16152 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16153 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16154 iRegI_R0 result, rFlagsReg cr)
16155 %{
16156 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16157 match(Set result (EncodeISOArray src (Binary dst len)));
16158 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16159 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16160
16161 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16162 ins_encode %{
16163 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16164 $result$$Register, false,
16165 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16166 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16167 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16168 %}
16169 ins_pipe(pipe_class_memory);
16170 %}
16171
16172 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16173 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16174 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16175 iRegI_R0 result, rFlagsReg cr)
16176 %{
16177 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16178 match(Set result (EncodeISOArray src (Binary dst len)));
16179 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16180 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16181
16182 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16183 ins_encode %{
16184 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16185 $result$$Register, true,
16186 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16187 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16188 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16189 %}
16190 ins_pipe(pipe_class_memory);
16191 %}
16192
16193 //----------------------------- CompressBits/ExpandBits ------------------------
16194
16195 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16196 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16197 match(Set dst (CompressBits src mask));
16198 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16199 format %{ "mov $tsrc, $src\n\t"
16200 "mov $tmask, $mask\n\t"
16201 "bext $tdst, $tsrc, $tmask\n\t"
16202 "mov $dst, $tdst"
16203 %}
16204 ins_encode %{
16205 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16206 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16207 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16208 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16209 %}
16210 ins_pipe(pipe_slow);
16211 %}
16212
16213 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16214 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16215 match(Set dst (CompressBits (LoadI mem) mask));
16216 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16217 format %{ "ldrs $tsrc, $mem\n\t"
16218 "ldrs $tmask, $mask\n\t"
16219 "bext $tdst, $tsrc, $tmask\n\t"
16220 "mov $dst, $tdst"
16221 %}
16222 ins_encode %{
16223 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16224 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16225 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16226 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16227 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16228 %}
16229 ins_pipe(pipe_slow);
16230 %}
16231
16232 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16233 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16234 match(Set dst (CompressBits src mask));
16235 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16236 format %{ "mov $tsrc, $src\n\t"
16237 "mov $tmask, $mask\n\t"
16238 "bext $tdst, $tsrc, $tmask\n\t"
16239 "mov $dst, $tdst"
16240 %}
16241 ins_encode %{
16242 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16243 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16244 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16245 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16246 %}
16247 ins_pipe(pipe_slow);
16248 %}
16249
16250 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16251 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16252 match(Set dst (CompressBits (LoadL mem) mask));
16253 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16254 format %{ "ldrd $tsrc, $mem\n\t"
16255 "ldrd $tmask, $mask\n\t"
16256 "bext $tdst, $tsrc, $tmask\n\t"
16257 "mov $dst, $tdst"
16258 %}
16259 ins_encode %{
16260 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16261 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16262 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16263 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16264 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16265 %}
16266 ins_pipe(pipe_slow);
16267 %}
16268
16269 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16270 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16271 match(Set dst (ExpandBits src mask));
16272 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16273 format %{ "mov $tsrc, $src\n\t"
16274 "mov $tmask, $mask\n\t"
16275 "bdep $tdst, $tsrc, $tmask\n\t"
16276 "mov $dst, $tdst"
16277 %}
16278 ins_encode %{
16279 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16280 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16281 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16282 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16283 %}
16284 ins_pipe(pipe_slow);
16285 %}
16286
16287 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16288 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16289 match(Set dst (ExpandBits (LoadI mem) mask));
16290 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16291 format %{ "ldrs $tsrc, $mem\n\t"
16292 "ldrs $tmask, $mask\n\t"
16293 "bdep $tdst, $tsrc, $tmask\n\t"
16294 "mov $dst, $tdst"
16295 %}
16296 ins_encode %{
16297 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16298 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16299 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16300 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16301 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16302 %}
16303 ins_pipe(pipe_slow);
16304 %}
16305
16306 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16307 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16308 match(Set dst (ExpandBits src mask));
16309 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16310 format %{ "mov $tsrc, $src\n\t"
16311 "mov $tmask, $mask\n\t"
16312 "bdep $tdst, $tsrc, $tmask\n\t"
16313 "mov $dst, $tdst"
16314 %}
16315 ins_encode %{
16316 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16317 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16318 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16319 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16320 %}
16321 ins_pipe(pipe_slow);
16322 %}
16323
16324
16325 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16326 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16327 match(Set dst (ExpandBits (LoadL mem) mask));
16328 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16329 format %{ "ldrd $tsrc, $mem\n\t"
16330 "ldrd $tmask, $mask\n\t"
16331 "bdep $tdst, $tsrc, $tmask\n\t"
16332 "mov $dst, $tdst"
16333 %}
16334 ins_encode %{
16335 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16336 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16337 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16338 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16339 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16340 %}
16341 ins_pipe(pipe_slow);
16342 %}
16343
16344 //----------------------------- Reinterpret ----------------------------------
16345 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16346 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16347 match(Set dst (ReinterpretHF2S src));
16348 format %{ "reinterpretHF2S $dst, $src" %}
16349 ins_encode %{
16350 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16351 %}
16352 ins_pipe(pipe_slow);
16353 %}
16354
16355 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16356 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16357 match(Set dst (ReinterpretS2HF src));
16358 format %{ "reinterpretS2HF $dst, $src" %}
16359 ins_encode %{
16360 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16361 %}
16362 ins_pipe(pipe_slow);
16363 %}
16364
16365 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16366 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16367 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16368 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16369 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16370 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16371 // can be omitted in this pattern, resulting in -
16372 // fcvt $dst, $src // Convert float to half-precision float
16373 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16374 %{
16375 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16376 format %{ "convF2HFAndS2HF $dst, $src" %}
16377 ins_encode %{
16378 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16379 %}
16380 ins_pipe(pipe_slow);
16381 %}
16382
16383 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16384 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16385 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16386 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16387 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16388 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16389 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16390 // resulting in -
16391 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16392 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16393 %{
16394 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16395 format %{ "convHF2SAndHF2F $dst, $src" %}
16396 ins_encode %{
16397 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16398 %}
16399 ins_pipe(pipe_slow);
16400 %}
16401
16402 // ============================================================================
16403 // This name is KNOWN by the ADLC and cannot be changed.
16404 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16405 // for this guy.
16406 instruct tlsLoadP(thread_RegP dst)
16407 %{
16408 match(Set dst (ThreadLocal));
16409
16410 ins_cost(0);
16411
16412 format %{ " -- \t// $dst=Thread::current(), empty" %}
16413
16414 size(0);
16415
16416 ins_encode( /*empty*/ );
16417
16418 ins_pipe(pipe_class_empty);
16419 %}
16420
16421 //----------PEEPHOLE RULES-----------------------------------------------------
16422 // These must follow all instruction definitions as they use the names
16423 // defined in the instructions definitions.
16424 //
16425 // peepmatch ( root_instr_name [preceding_instruction]* );
16426 //
16427 // peepconstraint %{
16428 // (instruction_number.operand_name relational_op instruction_number.operand_name
16429 // [, ...] );
16430 // // instruction numbers are zero-based using left to right order in peepmatch
16431 //
16432 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16433 // // provide an instruction_number.operand_name for each operand that appears
16434 // // in the replacement instruction's match rule
16435 //
16436 // ---------VM FLAGS---------------------------------------------------------
16437 //
16438 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16439 //
16440 // Each peephole rule is given an identifying number starting with zero and
16441 // increasing by one in the order seen by the parser. An individual peephole
16442 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16443 // on the command-line.
16444 //
16445 // ---------CURRENT LIMITATIONS----------------------------------------------
16446 //
16447 // Only match adjacent instructions in same basic block
16448 // Only equality constraints
16449 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16450 // Only one replacement instruction
16451 //
16452 // ---------EXAMPLE----------------------------------------------------------
16453 //
16454 // // pertinent parts of existing instructions in architecture description
16455 // instruct movI(iRegINoSp dst, iRegI src)
16456 // %{
16457 // match(Set dst (CopyI src));
16458 // %}
16459 //
16460 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16461 // %{
16462 // match(Set dst (AddI dst src));
16463 // effect(KILL cr);
16464 // %}
16465 //
16466 // // Change (inc mov) to lea
16467 // peephole %{
16468 // // increment preceded by register-register move
16469 // peepmatch ( incI_iReg movI );
16470 // // require that the destination register of the increment
16471 // // match the destination register of the move
16472 // peepconstraint ( 0.dst == 1.dst );
16473 // // construct a replacement instruction that sets
16474 // // the destination to ( move's source register + one )
16475 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16476 // %}
16477 //
16478
16479 // Implementation no longer uses movX instructions since
16480 // machine-independent system no longer uses CopyX nodes.
16481 //
16482 // peephole
16483 // %{
16484 // peepmatch (incI_iReg movI);
16485 // peepconstraint (0.dst == 1.dst);
16486 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16487 // %}
16488
16489 // peephole
16490 // %{
16491 // peepmatch (decI_iReg movI);
16492 // peepconstraint (0.dst == 1.dst);
16493 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16494 // %}
16495
16496 // peephole
16497 // %{
16498 // peepmatch (addI_iReg_imm movI);
16499 // peepconstraint (0.dst == 1.dst);
16500 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16501 // %}
16502
16503 // peephole
16504 // %{
16505 // peepmatch (incL_iReg movL);
16506 // peepconstraint (0.dst == 1.dst);
16507 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16508 // %}
16509
16510 // peephole
16511 // %{
16512 // peepmatch (decL_iReg movL);
16513 // peepconstraint (0.dst == 1.dst);
16514 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16515 // %}
16516
16517 // peephole
16518 // %{
16519 // peepmatch (addL_iReg_imm movL);
16520 // peepconstraint (0.dst == 1.dst);
16521 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16522 // %}
16523
16524 // peephole
16525 // %{
16526 // peepmatch (addP_iReg_imm movP);
16527 // peepconstraint (0.dst == 1.dst);
16528 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16529 // %}
16530
16531 // // Change load of spilled value to only a spill
16532 // instruct storeI(memory mem, iRegI src)
16533 // %{
16534 // match(Set mem (StoreI mem src));
16535 // %}
16536 //
16537 // instruct loadI(iRegINoSp dst, memory mem)
16538 // %{
16539 // match(Set dst (LoadI mem));
16540 // %}
16541 //
16542
16543 //----------SMARTSPILL RULES---------------------------------------------------
16544 // These must follow all instruction definitions as they use the names
16545 // defined in the instructions definitions.
16546
16547 // Local Variables:
16548 // mode: c++
16549 // End: