1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2237 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2238 st->print_cr("\tcmpw rscratch1, r10");
2239 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2240 }
2241 #endif
2242
2243 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2244 {
2245 __ ic_check(InteriorEntryAlignment);
2246 }
2247
2248 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2249 {
2250 return MachNode::size(ra_);
2251 }
2252
2253 // REQUIRED EMIT CODE
2254
2255 //=============================================================================
2256
2257 // Emit deopt handler code.
2258 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2259 {
2260 // Note that the code buffer's insts_mark is always relative to insts.
2261 // That's why we must use the macroassembler to generate a handler.
2262 address base = __ start_a_stub(size_deopt_handler());
2263 if (base == nullptr) {
2264 ciEnv::current()->record_failure("CodeCache is full");
2265 return 0; // CodeBuffer::expand failed
2266 }
2267
2268 int offset = __ offset();
2269 Label start;
2270 __ bind(start);
2271 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2272
2273 int entry_offset = __ offset();
2274 __ b(start);
2275
2276 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2277 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2278 "out of bounds read in post-call NOP check");
2279 __ end_a_stub();
2280 return entry_offset;
2281 }
2282
2283 // REQUIRED MATCHER CODE
2284
2285 //=============================================================================
2286
2287 bool Matcher::match_rule_supported(int opcode) {
2288 if (!has_match_rule(opcode))
2289 return false;
2290
2291 switch (opcode) {
2292 case Op_OnSpinWait:
2293 return VM_Version::supports_on_spin_wait();
2294 case Op_CacheWB:
2295 case Op_CacheWBPreSync:
2296 case Op_CacheWBPostSync:
2297 if (!VM_Version::supports_data_cache_line_flush()) {
2298 return false;
2299 }
2300 break;
2301 case Op_ExpandBits:
2302 case Op_CompressBits:
2303 if (!VM_Version::supports_svebitperm()) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaF:
2308 case Op_FmaD:
2309 case Op_FmaVF:
2310 case Op_FmaVD:
2311 if (!UseFMA) {
2312 return false;
2313 }
2314 break;
2315 case Op_FmaHF:
2316 // UseFMA flag also needs to be checked along with FEAT_FP16
2317 if (!UseFMA || !is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 case Op_AddHF:
2322 case Op_SubHF:
2323 case Op_MulHF:
2324 case Op_DivHF:
2325 case Op_MinHF:
2326 case Op_MaxHF:
2327 case Op_SqrtHF:
2328 // Half-precision floating point scalar operations require FEAT_FP16
2329 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2330 // features are supported.
2331 if (!is_feat_fp16_supported()) {
2332 return false;
2333 }
2334 break;
2335 }
2336
2337 return true; // Per default match rules are supported.
2338 }
2339
2340 const RegMask* Matcher::predicate_reg_mask(void) {
2341 return &_PR_REG_mask;
2342 }
2343
2344 bool Matcher::supports_vector_calling_convention(void) {
2345 return EnableVectorSupport;
2346 }
2347
2348 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2349 assert(EnableVectorSupport, "sanity");
2350 int lo = V0_num;
2351 int hi = V0_H_num;
2352 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2353 hi = V0_K_num;
2354 }
2355 return OptoRegPair(hi, lo);
2356 }
2357
2358 // Is this branch offset short enough that a short branch can be used?
2359 //
2360 // NOTE: If the platform does not provide any short branch variants, then
2361 // this method should return false for offset 0.
2362 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2363 // The passed offset is relative to address of the branch.
2364
2365 return (-32768 <= offset && offset < 32768);
2366 }
2367
2368 // Vector width in bytes.
2369 int Matcher::vector_width_in_bytes(BasicType bt) {
2370 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2371 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2372 // Minimum 2 values in vector
2373 if (size < 2*type2aelembytes(bt)) size = 0;
2374 // But never < 4
2375 if (size < 4) size = 0;
2376 return size;
2377 }
2378
2379 // Limits on vector size (number of elements) loaded into vector.
2380 int Matcher::max_vector_size(const BasicType bt) {
2381 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2382 }
2383
2384 int Matcher::min_vector_size(const BasicType bt) {
2385 // Usually, the shortest vector length supported by AArch64 ISA and
2386 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2387 // vectors in a few special cases.
2388 int size;
2389 switch(bt) {
2390 case T_BOOLEAN:
2391 // Load/store a vector mask with only 2 elements for vector types
2392 // such as "2I/2F/2L/2D".
2393 size = 2;
2394 break;
2395 case T_BYTE:
2396 // Generate a "4B" vector, to support vector cast between "8B/16B"
2397 // and "4S/4I/4L/4F/4D".
2398 size = 4;
2399 break;
2400 case T_SHORT:
2401 // Generate a "2S" vector, to support vector cast between "4S/8S"
2402 // and "2I/2L/2F/2D".
2403 size = 2;
2404 break;
2405 default:
2406 // Limit the min vector length to 64-bit.
2407 size = 8 / type2aelembytes(bt);
2408 // The number of elements in a vector should be at least 2.
2409 size = MAX2(size, 2);
2410 }
2411
2412 int max_size = max_vector_size(bt);
2413 return MIN2(size, max_size);
2414 }
2415
2416 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2417 return Matcher::max_vector_size(bt);
2418 }
2419
2420 // Actual max scalable vector register length.
2421 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2422 return Matcher::max_vector_size(bt);
2423 }
2424
2425 // Vector ideal reg.
2426 uint Matcher::vector_ideal_reg(int len) {
2427 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2428 return Op_VecA;
2429 }
2430 switch(len) {
2431 // For 16-bit/32-bit mask vector, reuse VecD.
2432 case 2:
2433 case 4:
2434 case 8: return Op_VecD;
2435 case 16: return Op_VecX;
2436 }
2437 ShouldNotReachHere();
2438 return 0;
2439 }
2440
2441 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2442 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2443 switch (ideal_reg) {
2444 case Op_VecA: return new vecAOper();
2445 case Op_VecD: return new vecDOper();
2446 case Op_VecX: return new vecXOper();
2447 }
2448 ShouldNotReachHere();
2449 return nullptr;
2450 }
2451
2452 bool Matcher::is_reg2reg_move(MachNode* m) {
2453 return false;
2454 }
2455
2456 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2457 return false;
2458 }
2459
2460 bool Matcher::is_generic_vector(MachOper* opnd) {
2461 return opnd->opcode() == VREG;
2462 }
2463
2464 #ifdef ASSERT
2465 // Return whether or not this register is ever used as an argument.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486 #endif
2487
2488 uint Matcher::int_pressure_limit()
2489 {
2490 // JDK-8183543: When taking the number of available registers as int
2491 // register pressure threshold, the jtreg test:
2492 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2493 // failed due to C2 compilation failure with
2494 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2495 //
2496 // A derived pointer is live at CallNode and then is flagged by RA
2497 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2498 // derived pointers and lastly fail to spill after reaching maximum
2499 // number of iterations. Lowering the default pressure threshold to
2500 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2501 // a high register pressure area of the code so that split_DEF can
2502 // generate DefinitionSpillCopy for the derived pointer.
2503 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2504 if (!PreserveFramePointer) {
2505 // When PreserveFramePointer is off, frame pointer is allocatable,
2506 // but different from other SOC registers, it is excluded from
2507 // fatproj's mask because its save type is No-Save. Decrease 1 to
2508 // ensure high pressure at fatproj when PreserveFramePointer is off.
2509 // See check_pressure_at_fatproj().
2510 default_int_pressure_threshold--;
2511 }
2512 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2513 }
2514
2515 uint Matcher::float_pressure_limit()
2516 {
2517 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2518 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2519 }
2520
2521 const RegMask& Matcher::divI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for MODI projection of divmodI.
2527 const RegMask& Matcher::modI_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for DIVL projection of divmodL.
2533 const RegMask& Matcher::divL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 // Register for MODL projection of divmodL.
2539 const RegMask& Matcher::modL_proj_mask() {
2540 ShouldNotReachHere();
2541 return RegMask::EMPTY;
2542 }
2543
2544 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2545 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2546 Node* u = addp->fast_out(i);
2547 if (u->is_LoadStore()) {
2548 // On AArch64, LoadStoreNodes (i.e. compare and swap
2549 // instructions) only take register indirect as an operand, so
2550 // any attempt to use an AddPNode as an input to a LoadStoreNode
2551 // must fail.
2552 return false;
2553 }
2554 if (u->is_Mem()) {
2555 int opsize = u->as_Mem()->memory_size();
2556 assert(opsize > 0, "unexpected memory operand size");
2557 if (u->as_Mem()->memory_size() != (1<<shift)) {
2558 return false;
2559 }
2560 }
2561 }
2562 return true;
2563 }
2564
2565 // Convert BoolTest condition to Assembler condition.
2566 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2567 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2568 Assembler::Condition result;
2569 switch(cond) {
2570 case BoolTest::eq:
2571 result = Assembler::EQ; break;
2572 case BoolTest::ne:
2573 result = Assembler::NE; break;
2574 case BoolTest::le:
2575 result = Assembler::LE; break;
2576 case BoolTest::ge:
2577 result = Assembler::GE; break;
2578 case BoolTest::lt:
2579 result = Assembler::LT; break;
2580 case BoolTest::gt:
2581 result = Assembler::GT; break;
2582 case BoolTest::ule:
2583 result = Assembler::LS; break;
2584 case BoolTest::uge:
2585 result = Assembler::HS; break;
2586 case BoolTest::ult:
2587 result = Assembler::LO; break;
2588 case BoolTest::ugt:
2589 result = Assembler::HI; break;
2590 case BoolTest::overflow:
2591 result = Assembler::VS; break;
2592 case BoolTest::no_overflow:
2593 result = Assembler::VC; break;
2594 default:
2595 ShouldNotReachHere();
2596 return Assembler::Condition(-1);
2597 }
2598
2599 // Check conversion
2600 if (cond & BoolTest::unsigned_compare) {
2601 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2602 } else {
2603 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2604 }
2605
2606 return result;
2607 }
2608
2609 // Binary src (Replicate con)
2610 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2611 if (n == nullptr || m == nullptr) {
2612 return false;
2613 }
2614
2615 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2616 return false;
2617 }
2618
2619 Node* imm_node = m->in(1);
2620 if (!imm_node->is_Con()) {
2621 return false;
2622 }
2623
2624 const Type* t = imm_node->bottom_type();
2625 if (!(t->isa_int() || t->isa_long())) {
2626 return false;
2627 }
2628
2629 switch (n->Opcode()) {
2630 case Op_AndV:
2631 case Op_OrV:
2632 case Op_XorV: {
2633 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2634 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2635 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2636 }
2637 case Op_AddVB:
2638 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2639 case Op_AddVS:
2640 case Op_AddVI:
2641 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2642 case Op_AddVL:
2643 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2644 default:
2645 return false;
2646 }
2647 }
2648
2649 // (XorV src (Replicate m1))
2650 // (XorVMask src (MaskAll m1))
2651 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2652 if (n != nullptr && m != nullptr) {
2653 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2654 VectorNode::is_all_ones_vector(m);
2655 }
2656 return false;
2657 }
2658
2659 // Should the matcher clone input 'm' of node 'n'?
2660 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2661 if (is_vshift_con_pattern(n, m) ||
2662 is_vector_bitwise_not_pattern(n, m) ||
2663 is_valid_sve_arith_imm_pattern(n, m) ||
2664 is_encode_and_store_pattern(n, m)) {
2665 mstack.push(m, Visit);
2666 return true;
2667 }
2668 return false;
2669 }
2670
2671 // Should the Matcher clone shifts on addressing modes, expecting them
2672 // to be subsumed into complex addressing expressions or compute them
2673 // into registers?
2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2675
2676 // Loads and stores with indirect memory input (e.g., volatile loads and
2677 // stores) do not subsume the input into complex addressing expressions. If
2678 // the addressing expression is input to at least one such load or store, do
2679 // not clone the addressing expression. Query needs_acquiring_load and
2680 // needs_releasing_store as a proxy for indirect memory input, as it is not
2681 // possible to directly query for indirect memory input at this stage.
2682 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2683 Node* n = m->fast_out(i);
2684 if (n->is_Load() && needs_acquiring_load(n)) {
2685 return false;
2686 }
2687 if (n->is_Store() && needs_releasing_store(n)) {
2688 return false;
2689 }
2690 }
2691
2692 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2693 return true;
2694 }
2695
2696 Node *off = m->in(AddPNode::Offset);
2697 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2698 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2699 // Are there other uses besides address expressions?
2700 !is_visited(off)) {
2701 address_visited.set(off->_idx); // Flag as address_visited
2702 mstack.push(off->in(2), Visit);
2703 Node *conv = off->in(1);
2704 if (conv->Opcode() == Op_ConvI2L &&
2705 // Are there other uses besides address expressions?
2706 !is_visited(conv)) {
2707 address_visited.set(conv->_idx); // Flag as address_visited
2708 mstack.push(conv->in(1), Pre_Visit);
2709 } else {
2710 mstack.push(conv, Pre_Visit);
2711 }
2712 address_visited.test_set(m->_idx); // Flag as address_visited
2713 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2714 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2715 return true;
2716 } else if (off->Opcode() == Op_ConvI2L &&
2717 // Are there other uses besides address expressions?
2718 !is_visited(off)) {
2719 address_visited.test_set(m->_idx); // Flag as address_visited
2720 address_visited.set(off->_idx); // Flag as address_visited
2721 mstack.push(off->in(1), Pre_Visit);
2722 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2723 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2724 return true;
2725 }
2726 return false;
2727 }
2728
2729 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2730 { \
2731 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2732 guarantee(DISP == 0, "mode not permitted for volatile"); \
2733 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2734 __ INSN(REG, as_Register(BASE)); \
2735 }
2736
2737
2738 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2739 {
2740 Address::extend scale;
2741
2742 // Hooboy, this is fugly. We need a way to communicate to the
2743 // encoder that the index needs to be sign extended, so we have to
2744 // enumerate all the cases.
2745 switch (opcode) {
2746 case INDINDEXSCALEDI2L:
2747 case INDINDEXSCALEDI2LN:
2748 case INDINDEXI2L:
2749 case INDINDEXI2LN:
2750 scale = Address::sxtw(size);
2751 break;
2752 default:
2753 scale = Address::lsl(size);
2754 }
2755
2756 if (index == -1) {
2757 return Address(base, disp);
2758 } else {
2759 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2760 return Address(base, as_Register(index), scale);
2761 }
2762 }
2763
2764
2765 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2766 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2767 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2768 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2769 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2770
2771 // Used for all non-volatile memory accesses. The use of
2772 // $mem->opcode() to discover whether this pattern uses sign-extended
2773 // offsets is something of a kludge.
2774 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2775 Register reg, int opcode,
2776 Register base, int index, int scale, int disp,
2777 int size_in_memory)
2778 {
2779 Address addr = mem2address(opcode, base, index, scale, disp);
2780 if (addr.getMode() == Address::base_plus_offset) {
2781 /* Fix up any out-of-range offsets. */
2782 assert_different_registers(rscratch1, base);
2783 assert_different_registers(rscratch1, reg);
2784 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2785 }
2786 (masm->*insn)(reg, addr);
2787 }
2788
2789 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2790 FloatRegister reg, int opcode,
2791 Register base, int index, int size, int disp,
2792 int size_in_memory)
2793 {
2794 Address::extend scale;
2795
2796 switch (opcode) {
2797 case INDINDEXSCALEDI2L:
2798 case INDINDEXSCALEDI2LN:
2799 scale = Address::sxtw(size);
2800 break;
2801 default:
2802 scale = Address::lsl(size);
2803 }
2804
2805 if (index == -1) {
2806 // Fix up any out-of-range offsets.
2807 assert_different_registers(rscratch1, base);
2808 Address addr = Address(base, disp);
2809 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2810 (masm->*insn)(reg, addr);
2811 } else {
2812 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2813 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2814 }
2815 }
2816
2817 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2818 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2819 int opcode, Register base, int index, int size, int disp)
2820 {
2821 if (index == -1) {
2822 (masm->*insn)(reg, T, Address(base, disp));
2823 } else {
2824 assert(disp == 0, "unsupported address mode");
2825 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2826 }
2827 }
2828
2829 %}
2830
2831
2832
2833 //----------ENCODING BLOCK-----------------------------------------------------
2834 // This block specifies the encoding classes used by the compiler to
2835 // output byte streams. Encoding classes are parameterized macros
2836 // used by Machine Instruction Nodes in order to generate the bit
2837 // encoding of the instruction. Operands specify their base encoding
2838 // interface with the interface keyword. There are currently
2839 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2840 // COND_INTER. REG_INTER causes an operand to generate a function
2841 // which returns its register number when queried. CONST_INTER causes
2842 // an operand to generate a function which returns the value of the
2843 // constant when queried. MEMORY_INTER causes an operand to generate
2844 // four functions which return the Base Register, the Index Register,
2845 // the Scale Value, and the Offset Value of the operand when queried.
2846 // COND_INTER causes an operand to generate six functions which return
2847 // the encoding code (ie - encoding bits for the instruction)
2848 // associated with each basic boolean condition for a conditional
2849 // instruction.
2850 //
2851 // Instructions specify two basic values for encoding. Again, a
2852 // function is available to check if the constant displacement is an
2853 // oop. They use the ins_encode keyword to specify their encoding
2854 // classes (which must be a sequence of enc_class names, and their
2855 // parameters, specified in the encoding block), and they use the
2856 // opcode keyword to specify, in order, their primary, secondary, and
2857 // tertiary opcode. Only the opcode sections which a particular
2858 // instruction needs for encoding need to be specified.
2859 encode %{
2860 // Build emit functions for each basic byte or larger field in the
2861 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2862 // from C++ code in the enc_class source block. Emit functions will
2863 // live in the main source block for now. In future, we can
2864 // generalize this by adding a syntax that specifies the sizes of
2865 // fields in an order, so that the adlc can build the emit functions
2866 // automagically
2867
2868 // catch all for unimplemented encodings
2869 enc_class enc_unimplemented %{
2870 __ unimplemented("C2 catch all");
2871 %}
2872
2873 // BEGIN Non-volatile memory access
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2974 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2982 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2990 Register src_reg = as_Register($src$$reg);
2991 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strb0(memory1 mem) %{
2998 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3000 %}
3001
3002 // This encoding class is generated automatically from ad_encode.m4.
3003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3004 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3005 Register src_reg = as_Register($src$$reg);
3006 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_strh0(memory2 mem) %{
3013 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3014 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3015 %}
3016
3017 // This encoding class is generated automatically from ad_encode.m4.
3018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3019 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3020 Register src_reg = as_Register($src$$reg);
3021 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strw0(memory4 mem) %{
3028 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3029 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3030 %}
3031
3032 // This encoding class is generated automatically from ad_encode.m4.
3033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3034 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3035 Register src_reg = as_Register($src$$reg);
3036 // we sometimes get asked to store the stack pointer into the
3037 // current thread -- we cannot do that directly on AArch64
3038 if (src_reg == r31_sp) {
3039 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3040 __ mov(rscratch2, sp);
3041 src_reg = rscratch2;
3042 }
3043 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3044 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3045 %}
3046
3047 // This encoding class is generated automatically from ad_encode.m4.
3048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3049 enc_class aarch64_enc_str0(memory8 mem) %{
3050 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3051 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3052 %}
3053
3054 // This encoding class is generated automatically from ad_encode.m4.
3055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3056 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3057 FloatRegister src_reg = as_FloatRegister($src$$reg);
3058 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3065 FloatRegister src_reg = as_FloatRegister($src$$reg);
3066 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3073 __ membar(Assembler::StoreStore);
3074 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3076 %}
3077
3078 // END Non-volatile memory access
3079
3080 // Vector loads and stores
3081 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3082 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3083 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3084 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3085 %}
3086
3087 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3088 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3089 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3090 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3091 %}
3092
3093 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3100 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3101 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3106 FloatRegister src_reg = as_FloatRegister($src$$reg);
3107 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3112 FloatRegister src_reg = as_FloatRegister($src$$reg);
3113 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3124 FloatRegister src_reg = as_FloatRegister($src$$reg);
3125 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 // volatile loads and stores
3130
3131 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3132 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3133 rscratch1, stlrb);
3134 %}
3135
3136 enc_class aarch64_enc_stlrb0(memory mem) %{
3137 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, stlrb);
3139 %}
3140
3141 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3142 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, stlrh);
3144 %}
3145
3146 enc_class aarch64_enc_stlrh0(memory mem) %{
3147 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, stlrh);
3149 %}
3150
3151 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrw);
3154 %}
3155
3156 enc_class aarch64_enc_stlrw0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrw);
3159 %}
3160
3161 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3162 Register dst_reg = as_Register($dst$$reg);
3163 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldarb);
3165 __ sxtbw(dst_reg, dst_reg);
3166 %}
3167
3168 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3169 Register dst_reg = as_Register($dst$$reg);
3170 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarb);
3172 __ sxtb(dst_reg, dst_reg);
3173 %}
3174
3175 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3176 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, ldarb);
3178 %}
3179
3180 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3181 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarb);
3183 %}
3184
3185 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3186 Register dst_reg = as_Register($dst$$reg);
3187 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarh);
3189 __ sxthw(dst_reg, dst_reg);
3190 %}
3191
3192 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3193 Register dst_reg = as_Register($dst$$reg);
3194 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarh);
3196 __ sxth(dst_reg, dst_reg);
3197 %}
3198
3199 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3200 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3201 rscratch1, ldarh);
3202 %}
3203
3204 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3205 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, ldarh);
3207 %}
3208
3209 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3210 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarw);
3212 %}
3213
3214 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3215 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarw);
3217 %}
3218
3219 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldar);
3222 %}
3223
3224 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3225 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarw);
3227 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3228 %}
3229
3230 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3231 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldar);
3233 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3234 %}
3235
3236 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3237 Register src_reg = as_Register($src$$reg);
3238 // we sometimes get asked to store the stack pointer into the
3239 // current thread -- we cannot do that directly on AArch64
3240 if (src_reg == r31_sp) {
3241 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3242 __ mov(rscratch2, sp);
3243 src_reg = rscratch2;
3244 }
3245 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3246 rscratch1, stlr);
3247 %}
3248
3249 enc_class aarch64_enc_stlr0(memory mem) %{
3250 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3251 rscratch1, stlr);
3252 %}
3253
3254 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3255 {
3256 FloatRegister src_reg = as_FloatRegister($src$$reg);
3257 __ fmovs(rscratch2, src_reg);
3258 }
3259 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3260 rscratch1, stlrw);
3261 %}
3262
3263 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3264 {
3265 FloatRegister src_reg = as_FloatRegister($src$$reg);
3266 __ fmovd(rscratch2, src_reg);
3267 }
3268 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3269 rscratch1, stlr);
3270 %}
3271
3272 // synchronized read/update encodings
3273
3274 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3275 Register dst_reg = as_Register($dst$$reg);
3276 Register base = as_Register($mem$$base);
3277 int index = $mem$$index;
3278 int scale = $mem$$scale;
3279 int disp = $mem$$disp;
3280 if (index == -1) {
3281 if (disp != 0) {
3282 __ lea(rscratch1, Address(base, disp));
3283 __ ldaxr(dst_reg, rscratch1);
3284 } else {
3285 // TODO
3286 // should we ever get anything other than this case?
3287 __ ldaxr(dst_reg, base);
3288 }
3289 } else {
3290 Register index_reg = as_Register(index);
3291 if (disp == 0) {
3292 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3293 __ ldaxr(dst_reg, rscratch1);
3294 } else {
3295 __ lea(rscratch1, Address(base, disp));
3296 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3297 __ ldaxr(dst_reg, rscratch1);
3298 }
3299 }
3300 %}
3301
3302 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3303 Register src_reg = as_Register($src$$reg);
3304 Register base = as_Register($mem$$base);
3305 int index = $mem$$index;
3306 int scale = $mem$$scale;
3307 int disp = $mem$$disp;
3308 if (index == -1) {
3309 if (disp != 0) {
3310 __ lea(rscratch2, Address(base, disp));
3311 __ stlxr(rscratch1, src_reg, rscratch2);
3312 } else {
3313 // TODO
3314 // should we ever get anything other than this case?
3315 __ stlxr(rscratch1, src_reg, base);
3316 }
3317 } else {
3318 Register index_reg = as_Register(index);
3319 if (disp == 0) {
3320 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3321 __ stlxr(rscratch1, src_reg, rscratch2);
3322 } else {
3323 __ lea(rscratch2, Address(base, disp));
3324 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3325 __ stlxr(rscratch1, src_reg, rscratch2);
3326 }
3327 }
3328 __ cmpw(rscratch1, zr);
3329 %}
3330
3331 // prefetch encodings
3332
3333 enc_class aarch64_enc_prefetchw(memory mem) %{
3334 Register base = as_Register($mem$$base);
3335 int index = $mem$$index;
3336 int scale = $mem$$scale;
3337 int disp = $mem$$disp;
3338 if (index == -1) {
3339 // Fix up any out-of-range offsets.
3340 assert_different_registers(rscratch1, base);
3341 Address addr = Address(base, disp);
3342 addr = __ legitimize_address(addr, 8, rscratch1);
3343 __ prfm(addr, PSTL1KEEP);
3344 } else {
3345 Register index_reg = as_Register(index);
3346 if (disp == 0) {
3347 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3348 } else {
3349 __ lea(rscratch1, Address(base, disp));
3350 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3351 }
3352 }
3353 %}
3354
3355 // mov encodings
3356
3357 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3358 uint32_t con = (uint32_t)$src$$constant;
3359 Register dst_reg = as_Register($dst$$reg);
3360 if (con == 0) {
3361 __ movw(dst_reg, zr);
3362 } else {
3363 __ movw(dst_reg, con);
3364 }
3365 %}
3366
3367 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3368 Register dst_reg = as_Register($dst$$reg);
3369 uint64_t con = (uint64_t)$src$$constant;
3370 if (con == 0) {
3371 __ mov(dst_reg, zr);
3372 } else {
3373 __ mov(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 address con = (address)$src$$constant;
3380 if (con == nullptr || con == (address)1) {
3381 ShouldNotReachHere();
3382 } else {
3383 relocInfo::relocType rtype = $src->constant_reloc();
3384 if (rtype == relocInfo::oop_type) {
3385 __ movoop(dst_reg, (jobject)con);
3386 } else if (rtype == relocInfo::metadata_type) {
3387 __ mov_metadata(dst_reg, (Metadata*)con);
3388 } else {
3389 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3390 // load fake address constants using a normal move
3391 if (! __ is_valid_AArch64_address(con) ||
3392 con < (address)(uintptr_t)os::vm_page_size()) {
3393 __ mov(dst_reg, con);
3394 } else {
3395 // no reloc so just use adrp and add
3396 uint64_t offset;
3397 __ adrp(dst_reg, con, offset);
3398 __ add(dst_reg, dst_reg, offset);
3399 }
3400 }
3401 }
3402 %}
3403
3404 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3405 Register dst_reg = as_Register($dst$$reg);
3406 __ mov(dst_reg, zr);
3407 %}
3408
3409 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3410 Register dst_reg = as_Register($dst$$reg);
3411 __ mov(dst_reg, (uint64_t)1);
3412 %}
3413
3414 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3415 Register dst_reg = as_Register($dst$$reg);
3416 address con = (address)$src$$constant;
3417 if (con == nullptr) {
3418 ShouldNotReachHere();
3419 } else {
3420 relocInfo::relocType rtype = $src->constant_reloc();
3421 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3422 __ set_narrow_oop(dst_reg, (jobject)con);
3423 }
3424 %}
3425
3426 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, zr);
3429 %}
3430
3431 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3432 Register dst_reg = as_Register($dst$$reg);
3433 address con = (address)$src$$constant;
3434 if (con == nullptr) {
3435 ShouldNotReachHere();
3436 } else {
3437 relocInfo::relocType rtype = $src->constant_reloc();
3438 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3439 __ set_narrow_klass(dst_reg, (Klass *)con);
3440 }
3441 %}
3442
3443 // arithmetic encodings
3444
3445 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3446 Register dst_reg = as_Register($dst$$reg);
3447 Register src_reg = as_Register($src1$$reg);
3448 int32_t con = (int32_t)$src2$$constant;
3449 // add has primary == 0, subtract has primary == 1
3450 if ($primary) { con = -con; }
3451 if (con < 0) {
3452 __ subw(dst_reg, src_reg, -con);
3453 } else {
3454 __ addw(dst_reg, src_reg, con);
3455 }
3456 %}
3457
3458 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3459 Register dst_reg = as_Register($dst$$reg);
3460 Register src_reg = as_Register($src1$$reg);
3461 int32_t con = (int32_t)$src2$$constant;
3462 // add has primary == 0, subtract has primary == 1
3463 if ($primary) { con = -con; }
3464 if (con < 0) {
3465 __ sub(dst_reg, src_reg, -con);
3466 } else {
3467 __ add(dst_reg, src_reg, con);
3468 }
3469 %}
3470
3471 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3472 Register dst_reg = as_Register($dst$$reg);
3473 Register src1_reg = as_Register($src1$$reg);
3474 Register src2_reg = as_Register($src2$$reg);
3475 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3476 %}
3477
3478 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3479 Register dst_reg = as_Register($dst$$reg);
3480 Register src1_reg = as_Register($src1$$reg);
3481 Register src2_reg = as_Register($src2$$reg);
3482 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3483 %}
3484
3485 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3486 Register dst_reg = as_Register($dst$$reg);
3487 Register src1_reg = as_Register($src1$$reg);
3488 Register src2_reg = as_Register($src2$$reg);
3489 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3490 %}
3491
3492 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3497 %}
3498
3499 // compare instruction encodings
3500
3501 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3502 Register reg1 = as_Register($src1$$reg);
3503 Register reg2 = as_Register($src2$$reg);
3504 __ cmpw(reg1, reg2);
3505 %}
3506
3507 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3508 Register reg = as_Register($src1$$reg);
3509 int32_t val = $src2$$constant;
3510 if (val >= 0) {
3511 __ subsw(zr, reg, val);
3512 } else {
3513 __ addsw(zr, reg, -val);
3514 }
3515 %}
3516
3517 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3518 Register reg1 = as_Register($src1$$reg);
3519 uint32_t val = (uint32_t)$src2$$constant;
3520 __ movw(rscratch1, val);
3521 __ cmpw(reg1, rscratch1);
3522 %}
3523
3524 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3525 Register reg1 = as_Register($src1$$reg);
3526 Register reg2 = as_Register($src2$$reg);
3527 __ cmp(reg1, reg2);
3528 %}
3529
3530 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3531 Register reg = as_Register($src1$$reg);
3532 int64_t val = $src2$$constant;
3533 if (val >= 0) {
3534 __ subs(zr, reg, val);
3535 } else if (val != -val) {
3536 __ adds(zr, reg, -val);
3537 } else {
3538 // aargh, Long.MIN_VALUE is a special case
3539 __ orr(rscratch1, zr, (uint64_t)val);
3540 __ subs(zr, reg, rscratch1);
3541 }
3542 %}
3543
3544 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3545 Register reg1 = as_Register($src1$$reg);
3546 uint64_t val = (uint64_t)$src2$$constant;
3547 __ mov(rscratch1, val);
3548 __ cmp(reg1, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3552 Register reg1 = as_Register($src1$$reg);
3553 Register reg2 = as_Register($src2$$reg);
3554 __ cmp(reg1, reg2);
3555 %}
3556
3557 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 Register reg2 = as_Register($src2$$reg);
3560 __ cmpw(reg1, reg2);
3561 %}
3562
3563 enc_class aarch64_enc_testp(iRegP src) %{
3564 Register reg = as_Register($src$$reg);
3565 __ cmp(reg, zr);
3566 %}
3567
3568 enc_class aarch64_enc_testn(iRegN src) %{
3569 Register reg = as_Register($src$$reg);
3570 __ cmpw(reg, zr);
3571 %}
3572
3573 enc_class aarch64_enc_b(label lbl) %{
3574 Label *L = $lbl$$label;
3575 __ b(*L);
3576 %}
3577
3578 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3579 Label *L = $lbl$$label;
3580 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3581 %}
3582
3583 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3584 Label *L = $lbl$$label;
3585 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3586 %}
3587
3588 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3589 %{
3590 Register sub_reg = as_Register($sub$$reg);
3591 Register super_reg = as_Register($super$$reg);
3592 Register temp_reg = as_Register($temp$$reg);
3593 Register result_reg = as_Register($result$$reg);
3594
3595 Label miss;
3596 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3597 nullptr, &miss,
3598 /*set_cond_codes:*/ true);
3599 if ($primary) {
3600 __ mov(result_reg, zr);
3601 }
3602 __ bind(miss);
3603 %}
3604
3605 enc_class aarch64_enc_java_static_call(method meth) %{
3606 address addr = (address)$meth$$method;
3607 address call;
3608 if (!_method) {
3609 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3610 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3611 if (call == nullptr) {
3612 ciEnv::current()->record_failure("CodeCache is full");
3613 return;
3614 }
3615 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3616 // The NOP here is purely to ensure that eliding a call to
3617 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3618 __ nop();
3619 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3620 } else {
3621 int method_index = resolved_method_index(masm);
3622 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3623 : static_call_Relocation::spec(method_index);
3624 call = __ trampoline_call(Address(addr, rspec));
3625 if (call == nullptr) {
3626 ciEnv::current()->record_failure("CodeCache is full");
3627 return;
3628 }
3629 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3630 // Calls of the same statically bound method can share
3631 // a stub to the interpreter.
3632 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3633 } else {
3634 // Emit stub for static call
3635 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3636 if (stub == nullptr) {
3637 ciEnv::current()->record_failure("CodeCache is full");
3638 return;
3639 }
3640 }
3641 }
3642
3643 __ post_call_nop();
3644
3645 // Only non uncommon_trap calls need to reinitialize ptrue.
3646 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3647 __ reinitialize_ptrue();
3648 }
3649 %}
3650
3651 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3652 int method_index = resolved_method_index(masm);
3653 address call = __ ic_call((address)$meth$$method, method_index);
3654 if (call == nullptr) {
3655 ciEnv::current()->record_failure("CodeCache is full");
3656 return;
3657 }
3658 __ post_call_nop();
3659 if (Compile::current()->max_vector_size() > 0) {
3660 __ reinitialize_ptrue();
3661 }
3662 %}
3663
3664 enc_class aarch64_enc_call_epilog() %{
3665 if (VerifyStackAtCalls) {
3666 // Check that stack depth is unchanged: find majik cookie on stack
3667 __ call_Unimplemented();
3668 }
3669 %}
3670
3671 enc_class aarch64_enc_java_to_runtime(method meth) %{
3672 // some calls to generated routines (arraycopy code) are scheduled
3673 // by C2 as runtime calls. if so we can call them using a br (they
3674 // will be in a reachable segment) otherwise we have to use a blr
3675 // which loads the absolute address into a register.
3676 address entry = (address)$meth$$method;
3677 CodeBlob *cb = CodeCache::find_blob(entry);
3678 if (cb) {
3679 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3680 if (call == nullptr) {
3681 ciEnv::current()->record_failure("CodeCache is full");
3682 return;
3683 }
3684 __ post_call_nop();
3685 } else {
3686 Label retaddr;
3687 // Make the anchor frame walkable
3688 __ adr(rscratch2, retaddr);
3689 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3690 __ lea(rscratch1, RuntimeAddress(entry));
3691 __ blr(rscratch1);
3692 __ bind(retaddr);
3693 __ post_call_nop();
3694 }
3695 if (Compile::current()->max_vector_size() > 0) {
3696 __ reinitialize_ptrue();
3697 }
3698 %}
3699
3700 enc_class aarch64_enc_rethrow() %{
3701 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3702 %}
3703
3704 enc_class aarch64_enc_ret() %{
3705 #ifdef ASSERT
3706 if (Compile::current()->max_vector_size() > 0) {
3707 __ verify_ptrue();
3708 }
3709 #endif
3710 __ ret(lr);
3711 %}
3712
3713 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3714 Register target_reg = as_Register($jump_target$$reg);
3715 __ br(target_reg);
3716 %}
3717
3718 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3719 Register target_reg = as_Register($jump_target$$reg);
3720 // exception oop should be in r0
3721 // ret addr has been popped into lr
3722 // callee expects it in r3
3723 __ mov(r3, lr);
3724 __ br(target_reg);
3725 %}
3726
3727 %}
3728
3729 //----------FRAME--------------------------------------------------------------
3730 // Definition of frame structure and management information.
3731 //
3732 // S T A C K L A Y O U T Allocators stack-slot number
3733 // | (to get allocators register number
3734 // G Owned by | | v add OptoReg::stack0())
3735 // r CALLER | |
3736 // o | +--------+ pad to even-align allocators stack-slot
3737 // w V | pad0 | numbers; owned by CALLER
3738 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3739 // h ^ | in | 5
3740 // | | args | 4 Holes in incoming args owned by SELF
3741 // | | | | 3
3742 // | | +--------+
3743 // V | | old out| Empty on Intel, window on Sparc
3744 // | old |preserve| Must be even aligned.
3745 // | SP-+--------+----> Matcher::_old_SP, even aligned
3746 // | | in | 3 area for Intel ret address
3747 // Owned by |preserve| Empty on Sparc.
3748 // SELF +--------+
3749 // | | pad2 | 2 pad to align old SP
3750 // | +--------+ 1
3751 // | | locks | 0
3752 // | +--------+----> OptoReg::stack0(), even aligned
3753 // | | pad1 | 11 pad to align new SP
3754 // | +--------+
3755 // | | | 10
3756 // | | spills | 9 spills
3757 // V | | 8 (pad0 slot for callee)
3758 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3759 // ^ | out | 7
3760 // | | args | 6 Holes in outgoing args owned by CALLEE
3761 // Owned by +--------+
3762 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3763 // | new |preserve| Must be even-aligned.
3764 // | SP-+--------+----> Matcher::_new_SP, even aligned
3765 // | | |
3766 //
3767 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3768 // known from SELF's arguments and the Java calling convention.
3769 // Region 6-7 is determined per call site.
3770 // Note 2: If the calling convention leaves holes in the incoming argument
3771 // area, those holes are owned by SELF. Holes in the outgoing area
3772 // are owned by the CALLEE. Holes should not be necessary in the
3773 // incoming area, as the Java calling convention is completely under
3774 // the control of the AD file. Doubles can be sorted and packed to
3775 // avoid holes. Holes in the outgoing arguments may be necessary for
3776 // varargs C calling conventions.
3777 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3778 // even aligned with pad0 as needed.
3779 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3780 // (the latter is true on Intel but is it false on AArch64?)
3781 // region 6-11 is even aligned; it may be padded out more so that
3782 // the region from SP to FP meets the minimum stack alignment.
3783 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3784 // alignment. Region 11, pad1, may be dynamically extended so that
3785 // SP meets the minimum alignment.
3786
3787 frame %{
3788 // These three registers define part of the calling convention
3789 // between compiled code and the interpreter.
3790
3791 // Inline Cache Register or Method for I2C.
3792 inline_cache_reg(R12);
3793
3794 // Number of stack slots consumed by locking an object
3795 sync_stack_slots(2);
3796
3797 // Compiled code's Frame Pointer
3798 frame_pointer(R31);
3799
3800 // Stack alignment requirement
3801 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3802
3803 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3804 // for calls to C. Supports the var-args backing area for register parms.
3805 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3806
3807 // The after-PROLOG location of the return address. Location of
3808 // return address specifies a type (REG or STACK) and a number
3809 // representing the register number (i.e. - use a register name) or
3810 // stack slot.
3811 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3812 // Otherwise, it is above the locks and verification slot and alignment word
3813 // TODO this may well be correct but need to check why that - 2 is there
3814 // ppc port uses 0 but we definitely need to allow for fixed_slots
3815 // which folds in the space used for monitors
3816 return_addr(STACK - 2 +
3817 align_up((Compile::current()->in_preserve_stack_slots() +
3818 Compile::current()->fixed_slots()),
3819 stack_alignment_in_slots()));
3820
3821 // Location of compiled Java return values. Same as C for now.
3822 return_value
3823 %{
3824 // TODO do we allow ideal_reg == Op_RegN???
3825 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3826 "only return normal values");
3827
3828 static const int lo[Op_RegL + 1] = { // enum name
3829 0, // Op_Node
3830 0, // Op_Set
3831 R0_num, // Op_RegN
3832 R0_num, // Op_RegI
3833 R0_num, // Op_RegP
3834 V0_num, // Op_RegF
3835 V0_num, // Op_RegD
3836 R0_num // Op_RegL
3837 };
3838
3839 static const int hi[Op_RegL + 1] = { // enum name
3840 0, // Op_Node
3841 0, // Op_Set
3842 OptoReg::Bad, // Op_RegN
3843 OptoReg::Bad, // Op_RegI
3844 R0_H_num, // Op_RegP
3845 OptoReg::Bad, // Op_RegF
3846 V0_H_num, // Op_RegD
3847 R0_H_num // Op_RegL
3848 };
3849
3850 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3851 %}
3852 %}
3853
3854 //----------ATTRIBUTES---------------------------------------------------------
3855 //----------Operand Attributes-------------------------------------------------
3856 op_attrib op_cost(1); // Required cost attribute
3857
3858 //----------Instruction Attributes---------------------------------------------
3859 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3860 ins_attrib ins_size(32); // Required size attribute (in bits)
3861 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3862 // a non-matching short branch variant
3863 // of some long branch?
3864 ins_attrib ins_alignment(4); // Required alignment attribute (must
3865 // be a power of 2) specifies the
3866 // alignment that some part of the
3867 // instruction (not necessarily the
3868 // start) requires. If > 1, a
3869 // compute_padding() function must be
3870 // provided for the instruction
3871
3872 // Whether this node is expanded during code emission into a sequence of
3873 // instructions and the first instruction can perform an implicit null check.
3874 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3875
3876 //----------OPERANDS-----------------------------------------------------------
3877 // Operand definitions must precede instruction definitions for correct parsing
3878 // in the ADLC because operands constitute user defined types which are used in
3879 // instruction definitions.
3880
3881 //----------Simple Operands----------------------------------------------------
3882
3883 // Integer operands 32 bit
3884 // 32 bit immediate
3885 operand immI()
3886 %{
3887 match(ConI);
3888
3889 op_cost(0);
3890 format %{ %}
3891 interface(CONST_INTER);
3892 %}
3893
3894 // 32 bit zero
3895 operand immI0()
3896 %{
3897 predicate(n->get_int() == 0);
3898 match(ConI);
3899
3900 op_cost(0);
3901 format %{ %}
3902 interface(CONST_INTER);
3903 %}
3904
3905 // 32 bit unit increment
3906 operand immI_1()
3907 %{
3908 predicate(n->get_int() == 1);
3909 match(ConI);
3910
3911 op_cost(0);
3912 format %{ %}
3913 interface(CONST_INTER);
3914 %}
3915
3916 // 32 bit unit decrement
3917 operand immI_M1()
3918 %{
3919 predicate(n->get_int() == -1);
3920 match(ConI);
3921
3922 op_cost(0);
3923 format %{ %}
3924 interface(CONST_INTER);
3925 %}
3926
3927 // Shift values for add/sub extension shift
3928 operand immIExt()
3929 %{
3930 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 operand immI_gt_1()
3939 %{
3940 predicate(n->get_int() > 1);
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 operand immI_le_4()
3949 %{
3950 predicate(n->get_int() <= 4);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_16()
3959 %{
3960 predicate(n->get_int() == 16);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_24()
3969 %{
3970 predicate(n->get_int() == 24);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_32()
3979 %{
3980 predicate(n->get_int() == 32);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_48()
3989 %{
3990 predicate(n->get_int() == 48);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_56()
3999 %{
4000 predicate(n->get_int() == 56);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_255()
4009 %{
4010 predicate(n->get_int() == 255);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_65535()
4019 %{
4020 predicate(n->get_int() == 65535);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_positive()
4029 %{
4030 predicate(n->get_int() > 0);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 // BoolTest condition for signed compare
4039 operand immI_cmp_cond()
4040 %{
4041 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4042 match(ConI);
4043
4044 op_cost(0);
4045 format %{ %}
4046 interface(CONST_INTER);
4047 %}
4048
4049 // BoolTest condition for unsigned compare
4050 operand immI_cmpU_cond()
4051 %{
4052 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 operand immL_255()
4061 %{
4062 predicate(n->get_long() == 255L);
4063 match(ConL);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immL_65535()
4071 %{
4072 predicate(n->get_long() == 65535L);
4073 match(ConL);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immL_4294967295()
4081 %{
4082 predicate(n->get_long() == 4294967295L);
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_bitmask()
4091 %{
4092 predicate((n->get_long() != 0)
4093 && ((n->get_long() & 0xc000000000000000l) == 0)
4094 && is_power_of_2(n->get_long() + 1));
4095 match(ConL);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 operand immI_bitmask()
4103 %{
4104 predicate((n->get_int() != 0)
4105 && ((n->get_int() & 0xc0000000) == 0)
4106 && is_power_of_2(n->get_int() + 1));
4107 match(ConI);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immL_positive_bitmaskI()
4115 %{
4116 predicate((n->get_long() != 0)
4117 && ((julong)n->get_long() < 0x80000000ULL)
4118 && is_power_of_2(n->get_long() + 1));
4119 match(ConL);
4120
4121 op_cost(0);
4122 format %{ %}
4123 interface(CONST_INTER);
4124 %}
4125
4126 // Scale values for scaled offset addressing modes (up to long but not quad)
4127 operand immIScale()
4128 %{
4129 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4130 match(ConI);
4131
4132 op_cost(0);
4133 format %{ %}
4134 interface(CONST_INTER);
4135 %}
4136
4137 // 5 bit signed integer
4138 operand immI5()
4139 %{
4140 predicate(Assembler::is_simm(n->get_int(), 5));
4141 match(ConI);
4142
4143 op_cost(0);
4144 format %{ %}
4145 interface(CONST_INTER);
4146 %}
4147
4148 // 7 bit unsigned integer
4149 operand immIU7()
4150 %{
4151 predicate(Assembler::is_uimm(n->get_int(), 7));
4152 match(ConI);
4153
4154 op_cost(0);
4155 format %{ %}
4156 interface(CONST_INTER);
4157 %}
4158
4159 // Offset for scaled or unscaled immediate loads and stores
4160 operand immIOffset()
4161 %{
4162 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4163 match(ConI);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 operand immIOffset1()
4171 %{
4172 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immIOffset2()
4181 %{
4182 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immIOffset4()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset8()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset16()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immLOffset()
4221 %{
4222 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4223 match(ConL);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immLoffset1()
4231 %{
4232 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4233 match(ConL);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immLoffset2()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4243 match(ConL);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLoffset4()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset8()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset16()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 // 5 bit signed long integer
4281 operand immL5()
4282 %{
4283 predicate(Assembler::is_simm(n->get_long(), 5));
4284 match(ConL);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 // 7 bit unsigned long integer
4292 operand immLU7()
4293 %{
4294 predicate(Assembler::is_uimm(n->get_long(), 7));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 // 8 bit signed value.
4303 operand immI8()
4304 %{
4305 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4306 match(ConI);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 // 8 bit signed value (simm8), or #simm8 LSL 8.
4314 operand immIDupV()
4315 %{
4316 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4317 match(ConI);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 8 bit signed value (simm8), or #simm8 LSL 8.
4325 operand immLDupV()
4326 %{
4327 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4328 match(ConL);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 8 bit signed value (simm8), or #simm8 LSL 8.
4336 operand immHDupV()
4337 %{
4338 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4339 match(ConH);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit integer valid for vector add sub immediate
4347 operand immBAddSubV()
4348 %{
4349 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4350 match(ConI);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 // 32 bit integer valid for add sub immediate
4358 operand immIAddSub()
4359 %{
4360 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4361 match(ConI);
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 32 bit integer valid for vector add sub immediate
4368 operand immIAddSubV()
4369 %{
4370 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 32 bit unsigned integer valid for logical immediate
4379
4380 operand immBLog()
4381 %{
4382 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4383 match(ConI);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 operand immSLog()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4393 match(ConI);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 operand immILog()
4401 %{
4402 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 // Integer operands 64 bit
4411 // 64 bit immediate
4412 operand immL()
4413 %{
4414 match(ConL);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 64 bit zero
4422 operand immL0()
4423 %{
4424 predicate(n->get_long() == 0);
4425 match(ConL);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 // 64 bit unit decrement
4433 operand immL_M1()
4434 %{
4435 predicate(n->get_long() == -1);
4436 match(ConL);
4437
4438 op_cost(0);
4439 format %{ %}
4440 interface(CONST_INTER);
4441 %}
4442
4443 // 64 bit integer valid for add sub immediate
4444 operand immLAddSub()
4445 %{
4446 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4447 match(ConL);
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit integer valid for addv subv immediate
4454 operand immLAddSubV()
4455 %{
4456 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit integer valid for logical immediate
4465 operand immLLog()
4466 %{
4467 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4468 match(ConL);
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // Long Immediate: low 32-bit mask
4475 operand immL_32bits()
4476 %{
4477 predicate(n->get_long() == 0xFFFFFFFFL);
4478 match(ConL);
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // Pointer operands
4485 // Pointer Immediate
4486 operand immP()
4487 %{
4488 match(ConP);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // nullptr Pointer Immediate
4496 operand immP0()
4497 %{
4498 predicate(n->get_ptr() == 0);
4499 match(ConP);
4500
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 // Pointer Immediate One
4507 // this is used in object initialization (initial object header)
4508 operand immP_1()
4509 %{
4510 predicate(n->get_ptr() == 1);
4511 match(ConP);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // AOT Runtime Constants Address
4519 operand immAOTRuntimeConstantsAddress()
4520 %{
4521 // Check if the address is in the range of AOT Runtime Constants
4522 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4523 match(ConP);
4524
4525 op_cost(0);
4526 format %{ %}
4527 interface(CONST_INTER);
4528 %}
4529
4530 // Float and Double operands
4531 // Double Immediate
4532 operand immD()
4533 %{
4534 match(ConD);
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // Double Immediate: +0.0d
4541 operand immD0()
4542 %{
4543 predicate(jlong_cast(n->getd()) == 0);
4544 match(ConD);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 // constant 'double +0.0'.
4552 operand immDPacked()
4553 %{
4554 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4555 match(ConD);
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // Float Immediate
4562 operand immF()
4563 %{
4564 match(ConF);
4565 op_cost(0);
4566 format %{ %}
4567 interface(CONST_INTER);
4568 %}
4569
4570 // Float Immediate: +0.0f.
4571 operand immF0()
4572 %{
4573 predicate(jint_cast(n->getf()) == 0);
4574 match(ConF);
4575
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // Half Float (FP16) Immediate
4582 operand immH()
4583 %{
4584 match(ConH);
4585 op_cost(0);
4586 format %{ %}
4587 interface(CONST_INTER);
4588 %}
4589
4590 //
4591 operand immFPacked()
4592 %{
4593 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4594 match(ConF);
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 // Narrow pointer operands
4601 // Narrow Pointer Immediate
4602 operand immN()
4603 %{
4604 match(ConN);
4605
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // Narrow nullptr Pointer Immediate
4612 operand immN0()
4613 %{
4614 predicate(n->get_narrowcon() == 0);
4615 match(ConN);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 operand immNKlass()
4623 %{
4624 match(ConNKlass);
4625
4626 op_cost(0);
4627 format %{ %}
4628 interface(CONST_INTER);
4629 %}
4630
4631 // Integer 32 bit Register Operands
4632 // Integer 32 bitRegister (excludes SP)
4633 operand iRegI()
4634 %{
4635 constraint(ALLOC_IN_RC(any_reg32));
4636 match(RegI);
4637 match(iRegINoSp);
4638 op_cost(0);
4639 format %{ %}
4640 interface(REG_INTER);
4641 %}
4642
4643 // Integer 32 bit Register not Special
4644 operand iRegINoSp()
4645 %{
4646 constraint(ALLOC_IN_RC(no_special_reg32));
4647 match(RegI);
4648 op_cost(0);
4649 format %{ %}
4650 interface(REG_INTER);
4651 %}
4652
4653 // Integer 64 bit Register Operands
4654 // Integer 64 bit Register (includes SP)
4655 operand iRegL()
4656 %{
4657 constraint(ALLOC_IN_RC(any_reg));
4658 match(RegL);
4659 match(iRegLNoSp);
4660 op_cost(0);
4661 format %{ %}
4662 interface(REG_INTER);
4663 %}
4664
4665 // Integer 64 bit Register not Special
4666 operand iRegLNoSp()
4667 %{
4668 constraint(ALLOC_IN_RC(no_special_reg));
4669 match(RegL);
4670 match(iRegL_R0);
4671 format %{ %}
4672 interface(REG_INTER);
4673 %}
4674
4675 // Pointer Register Operands
4676 // Pointer Register
4677 operand iRegP()
4678 %{
4679 constraint(ALLOC_IN_RC(ptr_reg));
4680 match(RegP);
4681 match(iRegPNoSp);
4682 match(iRegP_R0);
4683 //match(iRegP_R2);
4684 //match(iRegP_R4);
4685 match(iRegP_R5);
4686 match(thread_RegP);
4687 op_cost(0);
4688 format %{ %}
4689 interface(REG_INTER);
4690 %}
4691
4692 // Pointer 64 bit Register not Special
4693 operand iRegPNoSp()
4694 %{
4695 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4696 match(RegP);
4697 // match(iRegP);
4698 // match(iRegP_R0);
4699 // match(iRegP_R2);
4700 // match(iRegP_R4);
4701 // match(iRegP_R5);
4702 // match(thread_RegP);
4703 op_cost(0);
4704 format %{ %}
4705 interface(REG_INTER);
4706 %}
4707
4708 // This operand is not allowed to use rfp even if
4709 // rfp is not used to hold the frame pointer.
4710 operand iRegPNoSpNoRfp()
4711 %{
4712 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4713 match(RegP);
4714 match(iRegPNoSp);
4715 op_cost(0);
4716 format %{ %}
4717 interface(REG_INTER);
4718 %}
4719
4720 // Pointer 64 bit Register R0 only
4721 operand iRegP_R0()
4722 %{
4723 constraint(ALLOC_IN_RC(r0_reg));
4724 match(RegP);
4725 // match(iRegP);
4726 match(iRegPNoSp);
4727 op_cost(0);
4728 format %{ %}
4729 interface(REG_INTER);
4730 %}
4731
4732 // Pointer 64 bit Register R1 only
4733 operand iRegP_R1()
4734 %{
4735 constraint(ALLOC_IN_RC(r1_reg));
4736 match(RegP);
4737 // match(iRegP);
4738 match(iRegPNoSp);
4739 op_cost(0);
4740 format %{ %}
4741 interface(REG_INTER);
4742 %}
4743
4744 // Pointer 64 bit Register R2 only
4745 operand iRegP_R2()
4746 %{
4747 constraint(ALLOC_IN_RC(r2_reg));
4748 match(RegP);
4749 // match(iRegP);
4750 match(iRegPNoSp);
4751 op_cost(0);
4752 format %{ %}
4753 interface(REG_INTER);
4754 %}
4755
4756 // Pointer 64 bit Register R3 only
4757 operand iRegP_R3()
4758 %{
4759 constraint(ALLOC_IN_RC(r3_reg));
4760 match(RegP);
4761 // match(iRegP);
4762 match(iRegPNoSp);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 // Pointer 64 bit Register R4 only
4769 operand iRegP_R4()
4770 %{
4771 constraint(ALLOC_IN_RC(r4_reg));
4772 match(RegP);
4773 // match(iRegP);
4774 match(iRegPNoSp);
4775 op_cost(0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer 64 bit Register R5 only
4781 operand iRegP_R5()
4782 %{
4783 constraint(ALLOC_IN_RC(r5_reg));
4784 match(RegP);
4785 // match(iRegP);
4786 match(iRegPNoSp);
4787 op_cost(0);
4788 format %{ %}
4789 interface(REG_INTER);
4790 %}
4791
4792 // Pointer 64 bit Register R10 only
4793 operand iRegP_R10()
4794 %{
4795 constraint(ALLOC_IN_RC(r10_reg));
4796 match(RegP);
4797 // match(iRegP);
4798 match(iRegPNoSp);
4799 op_cost(0);
4800 format %{ %}
4801 interface(REG_INTER);
4802 %}
4803
4804 // Long 64 bit Register R0 only
4805 operand iRegL_R0()
4806 %{
4807 constraint(ALLOC_IN_RC(r0_reg));
4808 match(RegL);
4809 match(iRegLNoSp);
4810 op_cost(0);
4811 format %{ %}
4812 interface(REG_INTER);
4813 %}
4814
4815 // Long 64 bit Register R11 only
4816 operand iRegL_R11()
4817 %{
4818 constraint(ALLOC_IN_RC(r11_reg));
4819 match(RegL);
4820 match(iRegLNoSp);
4821 op_cost(0);
4822 format %{ %}
4823 interface(REG_INTER);
4824 %}
4825
4826 // Register R0 only
4827 operand iRegI_R0()
4828 %{
4829 constraint(ALLOC_IN_RC(int_r0_reg));
4830 match(RegI);
4831 match(iRegINoSp);
4832 op_cost(0);
4833 format %{ %}
4834 interface(REG_INTER);
4835 %}
4836
4837 // Register R2 only
4838 operand iRegI_R2()
4839 %{
4840 constraint(ALLOC_IN_RC(int_r2_reg));
4841 match(RegI);
4842 match(iRegINoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Register R3 only
4849 operand iRegI_R3()
4850 %{
4851 constraint(ALLOC_IN_RC(int_r3_reg));
4852 match(RegI);
4853 match(iRegINoSp);
4854 op_cost(0);
4855 format %{ %}
4856 interface(REG_INTER);
4857 %}
4858
4859
4860 // Register R4 only
4861 operand iRegI_R4()
4862 %{
4863 constraint(ALLOC_IN_RC(int_r4_reg));
4864 match(RegI);
4865 match(iRegINoSp);
4866 op_cost(0);
4867 format %{ %}
4868 interface(REG_INTER);
4869 %}
4870
4871
4872 // Pointer Register Operands
4873 // Narrow Pointer Register
4874 operand iRegN()
4875 %{
4876 constraint(ALLOC_IN_RC(any_reg32));
4877 match(RegN);
4878 match(iRegNNoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884 // Integer 64 bit Register not Special
4885 operand iRegNNoSp()
4886 %{
4887 constraint(ALLOC_IN_RC(no_special_reg32));
4888 match(RegN);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Float Register
4895 // Float register operands
4896 operand vRegF()
4897 %{
4898 constraint(ALLOC_IN_RC(float_reg));
4899 match(RegF);
4900
4901 op_cost(0);
4902 format %{ %}
4903 interface(REG_INTER);
4904 %}
4905
4906 // Double Register
4907 // Double register operands
4908 operand vRegD()
4909 %{
4910 constraint(ALLOC_IN_RC(double_reg));
4911 match(RegD);
4912
4913 op_cost(0);
4914 format %{ %}
4915 interface(REG_INTER);
4916 %}
4917
4918 // Generic vector class. This will be used for
4919 // all vector operands, including NEON and SVE.
4920 operand vReg()
4921 %{
4922 constraint(ALLOC_IN_RC(dynamic));
4923 match(VecA);
4924 match(VecD);
4925 match(VecX);
4926
4927 op_cost(0);
4928 format %{ %}
4929 interface(REG_INTER);
4930 %}
4931
4932 operand vReg_V10()
4933 %{
4934 constraint(ALLOC_IN_RC(v10_veca_reg));
4935 match(vReg);
4936
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 operand vReg_V11()
4943 %{
4944 constraint(ALLOC_IN_RC(v11_veca_reg));
4945 match(vReg);
4946
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 operand vReg_V12()
4953 %{
4954 constraint(ALLOC_IN_RC(v12_veca_reg));
4955 match(vReg);
4956
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vReg_V13()
4963 %{
4964 constraint(ALLOC_IN_RC(v13_veca_reg));
4965 match(vReg);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vReg_V17()
4973 %{
4974 constraint(ALLOC_IN_RC(v17_veca_reg));
4975 match(vReg);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vReg_V18()
4983 %{
4984 constraint(ALLOC_IN_RC(v18_veca_reg));
4985 match(vReg);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V23()
4993 %{
4994 constraint(ALLOC_IN_RC(v23_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V24()
5003 %{
5004 constraint(ALLOC_IN_RC(v24_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vecA()
5013 %{
5014 constraint(ALLOC_IN_RC(vectora_reg));
5015 match(VecA);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vecD()
5023 %{
5024 constraint(ALLOC_IN_RC(vectord_reg));
5025 match(VecD);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vecX()
5033 %{
5034 constraint(ALLOC_IN_RC(vectorx_reg));
5035 match(VecX);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vRegD_V0()
5043 %{
5044 constraint(ALLOC_IN_RC(v0_reg));
5045 match(RegD);
5046 op_cost(0);
5047 format %{ %}
5048 interface(REG_INTER);
5049 %}
5050
5051 operand vRegD_V1()
5052 %{
5053 constraint(ALLOC_IN_RC(v1_reg));
5054 match(RegD);
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060 operand vRegD_V2()
5061 %{
5062 constraint(ALLOC_IN_RC(v2_reg));
5063 match(RegD);
5064 op_cost(0);
5065 format %{ %}
5066 interface(REG_INTER);
5067 %}
5068
5069 operand vRegD_V3()
5070 %{
5071 constraint(ALLOC_IN_RC(v3_reg));
5072 match(RegD);
5073 op_cost(0);
5074 format %{ %}
5075 interface(REG_INTER);
5076 %}
5077
5078 operand vRegD_V4()
5079 %{
5080 constraint(ALLOC_IN_RC(v4_reg));
5081 match(RegD);
5082 op_cost(0);
5083 format %{ %}
5084 interface(REG_INTER);
5085 %}
5086
5087 operand vRegD_V5()
5088 %{
5089 constraint(ALLOC_IN_RC(v5_reg));
5090 match(RegD);
5091 op_cost(0);
5092 format %{ %}
5093 interface(REG_INTER);
5094 %}
5095
5096 operand vRegD_V6()
5097 %{
5098 constraint(ALLOC_IN_RC(v6_reg));
5099 match(RegD);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105 operand vRegD_V7()
5106 %{
5107 constraint(ALLOC_IN_RC(v7_reg));
5108 match(RegD);
5109 op_cost(0);
5110 format %{ %}
5111 interface(REG_INTER);
5112 %}
5113
5114 operand vRegD_V12()
5115 %{
5116 constraint(ALLOC_IN_RC(v12_reg));
5117 match(RegD);
5118 op_cost(0);
5119 format %{ %}
5120 interface(REG_INTER);
5121 %}
5122
5123 operand vRegD_V13()
5124 %{
5125 constraint(ALLOC_IN_RC(v13_reg));
5126 match(RegD);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 operand pReg()
5133 %{
5134 constraint(ALLOC_IN_RC(pr_reg));
5135 match(RegVectMask);
5136 match(pRegGov);
5137 op_cost(0);
5138 format %{ %}
5139 interface(REG_INTER);
5140 %}
5141
5142 operand pRegGov()
5143 %{
5144 constraint(ALLOC_IN_RC(gov_pr));
5145 match(RegVectMask);
5146 match(pReg);
5147 op_cost(0);
5148 format %{ %}
5149 interface(REG_INTER);
5150 %}
5151
5152 operand pRegGov_P0()
5153 %{
5154 constraint(ALLOC_IN_RC(p0_reg));
5155 match(RegVectMask);
5156 op_cost(0);
5157 format %{ %}
5158 interface(REG_INTER);
5159 %}
5160
5161 operand pRegGov_P1()
5162 %{
5163 constraint(ALLOC_IN_RC(p1_reg));
5164 match(RegVectMask);
5165 op_cost(0);
5166 format %{ %}
5167 interface(REG_INTER);
5168 %}
5169
5170 // Flags register, used as output of signed compare instructions
5171
5172 // note that on AArch64 we also use this register as the output for
5173 // for floating point compare instructions (CmpF CmpD). this ensures
5174 // that ordered inequality tests use GT, GE, LT or LE none of which
5175 // pass through cases where the result is unordered i.e. one or both
5176 // inputs to the compare is a NaN. this means that the ideal code can
5177 // replace e.g. a GT with an LE and not end up capturing the NaN case
5178 // (where the comparison should always fail). EQ and NE tests are
5179 // always generated in ideal code so that unordered folds into the NE
5180 // case, matching the behaviour of AArch64 NE.
5181 //
5182 // This differs from x86 where the outputs of FP compares use a
5183 // special FP flags registers and where compares based on this
5184 // register are distinguished into ordered inequalities (cmpOpUCF) and
5185 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5186 // to explicitly handle the unordered case in branches. x86 also has
5187 // to include extra CMoveX rules to accept a cmpOpUCF input.
5188
5189 operand rFlagsReg()
5190 %{
5191 constraint(ALLOC_IN_RC(int_flags));
5192 match(RegFlags);
5193
5194 op_cost(0);
5195 format %{ "RFLAGS" %}
5196 interface(REG_INTER);
5197 %}
5198
5199 // Flags register, used as output of unsigned compare instructions
5200 operand rFlagsRegU()
5201 %{
5202 constraint(ALLOC_IN_RC(int_flags));
5203 match(RegFlags);
5204
5205 op_cost(0);
5206 format %{ "RFLAGSU" %}
5207 interface(REG_INTER);
5208 %}
5209
5210 // Special Registers
5211
5212 // Method Register
5213 operand inline_cache_RegP(iRegP reg)
5214 %{
5215 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5216 match(reg);
5217 match(iRegPNoSp);
5218 op_cost(0);
5219 format %{ %}
5220 interface(REG_INTER);
5221 %}
5222
5223 // Thread Register
5224 operand thread_RegP(iRegP reg)
5225 %{
5226 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5227 match(reg);
5228 op_cost(0);
5229 format %{ %}
5230 interface(REG_INTER);
5231 %}
5232
5233 //----------Memory Operands----------------------------------------------------
5234
5235 operand indirect(iRegP reg)
5236 %{
5237 constraint(ALLOC_IN_RC(ptr_reg));
5238 match(reg);
5239 op_cost(0);
5240 format %{ "[$reg]" %}
5241 interface(MEMORY_INTER) %{
5242 base($reg);
5243 index(0xffffffff);
5244 scale(0x0);
5245 disp(0x0);
5246 %}
5247 %}
5248
5249 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5250 %{
5251 constraint(ALLOC_IN_RC(ptr_reg));
5252 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5253 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5254 op_cost(0);
5255 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5256 interface(MEMORY_INTER) %{
5257 base($reg);
5258 index($ireg);
5259 scale($scale);
5260 disp(0x0);
5261 %}
5262 %}
5263
5264 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5265 %{
5266 constraint(ALLOC_IN_RC(ptr_reg));
5267 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5268 match(AddP reg (LShiftL lreg scale));
5269 op_cost(0);
5270 format %{ "$reg, $lreg lsl($scale)" %}
5271 interface(MEMORY_INTER) %{
5272 base($reg);
5273 index($lreg);
5274 scale($scale);
5275 disp(0x0);
5276 %}
5277 %}
5278
5279 operand indIndexI2L(iRegP reg, iRegI ireg)
5280 %{
5281 constraint(ALLOC_IN_RC(ptr_reg));
5282 match(AddP reg (ConvI2L ireg));
5283 op_cost(0);
5284 format %{ "$reg, $ireg, 0, I2L" %}
5285 interface(MEMORY_INTER) %{
5286 base($reg);
5287 index($ireg);
5288 scale(0x0);
5289 disp(0x0);
5290 %}
5291 %}
5292
5293 operand indIndex(iRegP reg, iRegL lreg)
5294 %{
5295 constraint(ALLOC_IN_RC(ptr_reg));
5296 match(AddP reg lreg);
5297 op_cost(0);
5298 format %{ "$reg, $lreg" %}
5299 interface(MEMORY_INTER) %{
5300 base($reg);
5301 index($lreg);
5302 scale(0x0);
5303 disp(0x0);
5304 %}
5305 %}
5306
5307 operand indOffI1(iRegP reg, immIOffset1 off)
5308 %{
5309 constraint(ALLOC_IN_RC(ptr_reg));
5310 match(AddP reg off);
5311 op_cost(0);
5312 format %{ "[$reg, $off]" %}
5313 interface(MEMORY_INTER) %{
5314 base($reg);
5315 index(0xffffffff);
5316 scale(0x0);
5317 disp($off);
5318 %}
5319 %}
5320
5321 operand indOffI2(iRegP reg, immIOffset2 off)
5322 %{
5323 constraint(ALLOC_IN_RC(ptr_reg));
5324 match(AddP reg off);
5325 op_cost(0);
5326 format %{ "[$reg, $off]" %}
5327 interface(MEMORY_INTER) %{
5328 base($reg);
5329 index(0xffffffff);
5330 scale(0x0);
5331 disp($off);
5332 %}
5333 %}
5334
5335 operand indOffI4(iRegP reg, immIOffset4 off)
5336 %{
5337 constraint(ALLOC_IN_RC(ptr_reg));
5338 match(AddP reg off);
5339 op_cost(0);
5340 format %{ "[$reg, $off]" %}
5341 interface(MEMORY_INTER) %{
5342 base($reg);
5343 index(0xffffffff);
5344 scale(0x0);
5345 disp($off);
5346 %}
5347 %}
5348
5349 operand indOffI8(iRegP reg, immIOffset8 off)
5350 %{
5351 constraint(ALLOC_IN_RC(ptr_reg));
5352 match(AddP reg off);
5353 op_cost(0);
5354 format %{ "[$reg, $off]" %}
5355 interface(MEMORY_INTER) %{
5356 base($reg);
5357 index(0xffffffff);
5358 scale(0x0);
5359 disp($off);
5360 %}
5361 %}
5362
5363 operand indOffI16(iRegP reg, immIOffset16 off)
5364 %{
5365 constraint(ALLOC_IN_RC(ptr_reg));
5366 match(AddP reg off);
5367 op_cost(0);
5368 format %{ "[$reg, $off]" %}
5369 interface(MEMORY_INTER) %{
5370 base($reg);
5371 index(0xffffffff);
5372 scale(0x0);
5373 disp($off);
5374 %}
5375 %}
5376
5377 operand indOffL1(iRegP reg, immLoffset1 off)
5378 %{
5379 constraint(ALLOC_IN_RC(ptr_reg));
5380 match(AddP reg off);
5381 op_cost(0);
5382 format %{ "[$reg, $off]" %}
5383 interface(MEMORY_INTER) %{
5384 base($reg);
5385 index(0xffffffff);
5386 scale(0x0);
5387 disp($off);
5388 %}
5389 %}
5390
5391 operand indOffL2(iRegP reg, immLoffset2 off)
5392 %{
5393 constraint(ALLOC_IN_RC(ptr_reg));
5394 match(AddP reg off);
5395 op_cost(0);
5396 format %{ "[$reg, $off]" %}
5397 interface(MEMORY_INTER) %{
5398 base($reg);
5399 index(0xffffffff);
5400 scale(0x0);
5401 disp($off);
5402 %}
5403 %}
5404
5405 operand indOffL4(iRegP reg, immLoffset4 off)
5406 %{
5407 constraint(ALLOC_IN_RC(ptr_reg));
5408 match(AddP reg off);
5409 op_cost(0);
5410 format %{ "[$reg, $off]" %}
5411 interface(MEMORY_INTER) %{
5412 base($reg);
5413 index(0xffffffff);
5414 scale(0x0);
5415 disp($off);
5416 %}
5417 %}
5418
5419 operand indOffL8(iRegP reg, immLoffset8 off)
5420 %{
5421 constraint(ALLOC_IN_RC(ptr_reg));
5422 match(AddP reg off);
5423 op_cost(0);
5424 format %{ "[$reg, $off]" %}
5425 interface(MEMORY_INTER) %{
5426 base($reg);
5427 index(0xffffffff);
5428 scale(0x0);
5429 disp($off);
5430 %}
5431 %}
5432
5433 operand indOffL16(iRegP reg, immLoffset16 off)
5434 %{
5435 constraint(ALLOC_IN_RC(ptr_reg));
5436 match(AddP reg off);
5437 op_cost(0);
5438 format %{ "[$reg, $off]" %}
5439 interface(MEMORY_INTER) %{
5440 base($reg);
5441 index(0xffffffff);
5442 scale(0x0);
5443 disp($off);
5444 %}
5445 %}
5446
5447 operand indirectX2P(iRegL reg)
5448 %{
5449 constraint(ALLOC_IN_RC(ptr_reg));
5450 match(CastX2P reg);
5451 op_cost(0);
5452 format %{ "[$reg]\t# long -> ptr" %}
5453 interface(MEMORY_INTER) %{
5454 base($reg);
5455 index(0xffffffff);
5456 scale(0x0);
5457 disp(0x0);
5458 %}
5459 %}
5460
5461 operand indOffX2P(iRegL reg, immLOffset off)
5462 %{
5463 constraint(ALLOC_IN_RC(ptr_reg));
5464 match(AddP (CastX2P reg) off);
5465 op_cost(0);
5466 format %{ "[$reg, $off]\t# long -> ptr" %}
5467 interface(MEMORY_INTER) %{
5468 base($reg);
5469 index(0xffffffff);
5470 scale(0x0);
5471 disp($off);
5472 %}
5473 %}
5474
5475 operand indirectN(iRegN reg)
5476 %{
5477 predicate(CompressedOops::shift() == 0);
5478 constraint(ALLOC_IN_RC(ptr_reg));
5479 match(DecodeN reg);
5480 op_cost(0);
5481 format %{ "[$reg]\t# narrow" %}
5482 interface(MEMORY_INTER) %{
5483 base($reg);
5484 index(0xffffffff);
5485 scale(0x0);
5486 disp(0x0);
5487 %}
5488 %}
5489
5490 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5491 %{
5492 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5493 constraint(ALLOC_IN_RC(ptr_reg));
5494 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5495 op_cost(0);
5496 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5497 interface(MEMORY_INTER) %{
5498 base($reg);
5499 index($ireg);
5500 scale($scale);
5501 disp(0x0);
5502 %}
5503 %}
5504
5505 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5506 %{
5507 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5508 constraint(ALLOC_IN_RC(ptr_reg));
5509 match(AddP (DecodeN reg) (LShiftL lreg scale));
5510 op_cost(0);
5511 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5512 interface(MEMORY_INTER) %{
5513 base($reg);
5514 index($lreg);
5515 scale($scale);
5516 disp(0x0);
5517 %}
5518 %}
5519
5520 operand indIndexI2LN(iRegN reg, iRegI ireg)
5521 %{
5522 predicate(CompressedOops::shift() == 0);
5523 constraint(ALLOC_IN_RC(ptr_reg));
5524 match(AddP (DecodeN reg) (ConvI2L ireg));
5525 op_cost(0);
5526 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5527 interface(MEMORY_INTER) %{
5528 base($reg);
5529 index($ireg);
5530 scale(0x0);
5531 disp(0x0);
5532 %}
5533 %}
5534
5535 operand indIndexN(iRegN reg, iRegL lreg)
5536 %{
5537 predicate(CompressedOops::shift() == 0);
5538 constraint(ALLOC_IN_RC(ptr_reg));
5539 match(AddP (DecodeN reg) lreg);
5540 op_cost(0);
5541 format %{ "$reg, $lreg\t# narrow" %}
5542 interface(MEMORY_INTER) %{
5543 base($reg);
5544 index($lreg);
5545 scale(0x0);
5546 disp(0x0);
5547 %}
5548 %}
5549
5550 operand indOffIN(iRegN reg, immIOffset off)
5551 %{
5552 predicate(CompressedOops::shift() == 0);
5553 constraint(ALLOC_IN_RC(ptr_reg));
5554 match(AddP (DecodeN reg) off);
5555 op_cost(0);
5556 format %{ "[$reg, $off]\t# narrow" %}
5557 interface(MEMORY_INTER) %{
5558 base($reg);
5559 index(0xffffffff);
5560 scale(0x0);
5561 disp($off);
5562 %}
5563 %}
5564
5565 operand indOffLN(iRegN reg, immLOffset off)
5566 %{
5567 predicate(CompressedOops::shift() == 0);
5568 constraint(ALLOC_IN_RC(ptr_reg));
5569 match(AddP (DecodeN reg) off);
5570 op_cost(0);
5571 format %{ "[$reg, $off]\t# narrow" %}
5572 interface(MEMORY_INTER) %{
5573 base($reg);
5574 index(0xffffffff);
5575 scale(0x0);
5576 disp($off);
5577 %}
5578 %}
5579
5580
5581 //----------Special Memory Operands--------------------------------------------
5582 // Stack Slot Operand - This operand is used for loading and storing temporary
5583 // values on the stack where a match requires a value to
5584 // flow through memory.
5585 operand stackSlotP(sRegP reg)
5586 %{
5587 constraint(ALLOC_IN_RC(stack_slots));
5588 op_cost(100);
5589 // No match rule because this operand is only generated in matching
5590 // match(RegP);
5591 format %{ "[$reg]" %}
5592 interface(MEMORY_INTER) %{
5593 base(0x1e); // RSP
5594 index(0x0); // No Index
5595 scale(0x0); // No Scale
5596 disp($reg); // Stack Offset
5597 %}
5598 %}
5599
5600 operand stackSlotI(sRegI reg)
5601 %{
5602 constraint(ALLOC_IN_RC(stack_slots));
5603 // No match rule because this operand is only generated in matching
5604 // match(RegI);
5605 format %{ "[$reg]" %}
5606 interface(MEMORY_INTER) %{
5607 base(0x1e); // RSP
5608 index(0x0); // No Index
5609 scale(0x0); // No Scale
5610 disp($reg); // Stack Offset
5611 %}
5612 %}
5613
5614 operand stackSlotF(sRegF reg)
5615 %{
5616 constraint(ALLOC_IN_RC(stack_slots));
5617 // No match rule because this operand is only generated in matching
5618 // match(RegF);
5619 format %{ "[$reg]" %}
5620 interface(MEMORY_INTER) %{
5621 base(0x1e); // RSP
5622 index(0x0); // No Index
5623 scale(0x0); // No Scale
5624 disp($reg); // Stack Offset
5625 %}
5626 %}
5627
5628 operand stackSlotD(sRegD reg)
5629 %{
5630 constraint(ALLOC_IN_RC(stack_slots));
5631 // No match rule because this operand is only generated in matching
5632 // match(RegD);
5633 format %{ "[$reg]" %}
5634 interface(MEMORY_INTER) %{
5635 base(0x1e); // RSP
5636 index(0x0); // No Index
5637 scale(0x0); // No Scale
5638 disp($reg); // Stack Offset
5639 %}
5640 %}
5641
5642 operand stackSlotL(sRegL reg)
5643 %{
5644 constraint(ALLOC_IN_RC(stack_slots));
5645 // No match rule because this operand is only generated in matching
5646 // match(RegL);
5647 format %{ "[$reg]" %}
5648 interface(MEMORY_INTER) %{
5649 base(0x1e); // RSP
5650 index(0x0); // No Index
5651 scale(0x0); // No Scale
5652 disp($reg); // Stack Offset
5653 %}
5654 %}
5655
5656 // Operands for expressing Control Flow
5657 // NOTE: Label is a predefined operand which should not be redefined in
5658 // the AD file. It is generically handled within the ADLC.
5659
5660 //----------Conditional Branch Operands----------------------------------------
5661 // Comparison Op - This is the operation of the comparison, and is limited to
5662 // the following set of codes:
5663 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5664 //
5665 // Other attributes of the comparison, such as unsignedness, are specified
5666 // by the comparison instruction that sets a condition code flags register.
5667 // That result is represented by a flags operand whose subtype is appropriate
5668 // to the unsignedness (etc.) of the comparison.
5669 //
5670 // Later, the instruction which matches both the Comparison Op (a Bool) and
5671 // the flags (produced by the Cmp) specifies the coding of the comparison op
5672 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5673
5674 // used for signed integral comparisons and fp comparisons
5675
5676 operand cmpOp()
5677 %{
5678 match(Bool);
5679
5680 format %{ "" %}
5681 interface(COND_INTER) %{
5682 equal(0x0, "eq");
5683 not_equal(0x1, "ne");
5684 less(0xb, "lt");
5685 greater_equal(0xa, "ge");
5686 less_equal(0xd, "le");
5687 greater(0xc, "gt");
5688 overflow(0x6, "vs");
5689 no_overflow(0x7, "vc");
5690 %}
5691 %}
5692
5693 // used for unsigned integral comparisons
5694
5695 operand cmpOpU()
5696 %{
5697 match(Bool);
5698
5699 format %{ "" %}
5700 interface(COND_INTER) %{
5701 equal(0x0, "eq");
5702 not_equal(0x1, "ne");
5703 less(0x3, "lo");
5704 greater_equal(0x2, "hs");
5705 less_equal(0x9, "ls");
5706 greater(0x8, "hi");
5707 overflow(0x6, "vs");
5708 no_overflow(0x7, "vc");
5709 %}
5710 %}
5711
5712 // used for certain integral comparisons which can be
5713 // converted to cbxx or tbxx instructions
5714
5715 operand cmpOpEqNe()
5716 %{
5717 match(Bool);
5718 op_cost(0);
5719 predicate(n->as_Bool()->_test._test == BoolTest::ne
5720 || n->as_Bool()->_test._test == BoolTest::eq);
5721
5722 format %{ "" %}
5723 interface(COND_INTER) %{
5724 equal(0x0, "eq");
5725 not_equal(0x1, "ne");
5726 less(0xb, "lt");
5727 greater_equal(0xa, "ge");
5728 less_equal(0xd, "le");
5729 greater(0xc, "gt");
5730 overflow(0x6, "vs");
5731 no_overflow(0x7, "vc");
5732 %}
5733 %}
5734
5735 // used for certain integral comparisons which can be
5736 // converted to cbxx or tbxx instructions
5737
5738 operand cmpOpLtGe()
5739 %{
5740 match(Bool);
5741 op_cost(0);
5742
5743 predicate(n->as_Bool()->_test._test == BoolTest::lt
5744 || n->as_Bool()->_test._test == BoolTest::ge);
5745
5746 format %{ "" %}
5747 interface(COND_INTER) %{
5748 equal(0x0, "eq");
5749 not_equal(0x1, "ne");
5750 less(0xb, "lt");
5751 greater_equal(0xa, "ge");
5752 less_equal(0xd, "le");
5753 greater(0xc, "gt");
5754 overflow(0x6, "vs");
5755 no_overflow(0x7, "vc");
5756 %}
5757 %}
5758
5759 // used for certain unsigned integral comparisons which can be
5760 // converted to cbxx or tbxx instructions
5761
5762 operand cmpOpUEqNeLeGt()
5763 %{
5764 match(Bool);
5765 op_cost(0);
5766
5767 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5768 n->as_Bool()->_test._test == BoolTest::ne ||
5769 n->as_Bool()->_test._test == BoolTest::le ||
5770 n->as_Bool()->_test._test == BoolTest::gt);
5771
5772 format %{ "" %}
5773 interface(COND_INTER) %{
5774 equal(0x0, "eq");
5775 not_equal(0x1, "ne");
5776 less(0x3, "lo");
5777 greater_equal(0x2, "hs");
5778 less_equal(0x9, "ls");
5779 greater(0x8, "hi");
5780 overflow(0x6, "vs");
5781 no_overflow(0x7, "vc");
5782 %}
5783 %}
5784
5785 // Special operand allowing long args to int ops to be truncated for free
5786
5787 operand iRegL2I(iRegL reg) %{
5788
5789 op_cost(0);
5790
5791 match(ConvL2I reg);
5792
5793 format %{ "l2i($reg)" %}
5794
5795 interface(REG_INTER)
5796 %}
5797
5798 operand iRegL2P(iRegL reg) %{
5799
5800 op_cost(0);
5801
5802 match(CastX2P reg);
5803
5804 format %{ "l2p($reg)" %}
5805
5806 interface(REG_INTER)
5807 %}
5808
5809 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5810 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5811 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5812 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5813
5814 //----------OPERAND CLASSES----------------------------------------------------
5815 // Operand Classes are groups of operands that are used as to simplify
5816 // instruction definitions by not requiring the AD writer to specify
5817 // separate instructions for every form of operand when the
5818 // instruction accepts multiple operand types with the same basic
5819 // encoding and format. The classic case of this is memory operands.
5820
5821 // memory is used to define read/write location for load/store
5822 // instruction defs. we can turn a memory op into an Address
5823
5824 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5825 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5826
5827 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5828 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5829
5830 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5831 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5832
5833 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5834 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5835
5836 // All of the memory operands. For the pipeline description.
5837 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5838 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5839 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5840
5841 opclass memory_noindex(indirect,
5842 indOffI1, indOffL1,indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5843 indirectN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5844
5845 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5846 // operations. it allows the src to be either an iRegI or a (ConvL2I
5847 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5848 // can be elided because the 32-bit instruction will just employ the
5849 // lower 32 bits anyway.
5850 //
5851 // n.b. this does not elide all L2I conversions. if the truncated
5852 // value is consumed by more than one operation then the ConvL2I
5853 // cannot be bundled into the consuming nodes so an l2i gets planted
5854 // (actually a movw $dst $src) and the downstream instructions consume
5855 // the result of the l2i as an iRegI input. That's a shame since the
5856 // movw is actually redundant but its not too costly.
5857
5858 opclass iRegIorL2I(iRegI, iRegL2I);
5859 opclass iRegPorL2P(iRegP, iRegL2P);
5860
5861 //----------PIPELINE-----------------------------------------------------------
5862 // Rules which define the behavior of the target architectures pipeline.
5863
5864 // For specific pipelines, eg A53, define the stages of that pipeline
5865 //pipe_desc(ISS, EX1, EX2, WR);
5866 #define ISS S0
5867 #define EX1 S1
5868 #define EX2 S2
5869 #define WR S3
5870
5871 // Integer ALU reg operation
5872 pipeline %{
5873
5874 attributes %{
5875 // ARM instructions are of fixed length
5876 fixed_size_instructions; // Fixed size instructions TODO does
5877 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5878 // ARM instructions come in 32-bit word units
5879 instruction_unit_size = 4; // An instruction is 4 bytes long
5880 instruction_fetch_unit_size = 64; // The processor fetches one line
5881 instruction_fetch_units = 1; // of 64 bytes
5882 %}
5883
5884 // We don't use an actual pipeline model so don't care about resources
5885 // or description. we do use pipeline classes to introduce fixed
5886 // latencies
5887
5888 //----------RESOURCES----------------------------------------------------------
5889 // Resources are the functional units available to the machine
5890
5891 resources( INS0, INS1, INS01 = INS0 | INS1,
5892 ALU0, ALU1, ALU = ALU0 | ALU1,
5893 MAC,
5894 DIV,
5895 BRANCH,
5896 LDST,
5897 NEON_FP);
5898
5899 //----------PIPELINE DESCRIPTION-----------------------------------------------
5900 // Pipeline Description specifies the stages in the machine's pipeline
5901
5902 // Define the pipeline as a generic 6 stage pipeline
5903 pipe_desc(S0, S1, S2, S3, S4, S5);
5904
5905 //----------PIPELINE CLASSES---------------------------------------------------
5906 // Pipeline Classes describe the stages in which input and output are
5907 // referenced by the hardware pipeline.
5908
5909 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5910 %{
5911 single_instruction;
5912 src1 : S1(read);
5913 src2 : S2(read);
5914 dst : S5(write);
5915 INS01 : ISS;
5916 NEON_FP : S5;
5917 %}
5918
5919 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5920 %{
5921 single_instruction;
5922 src1 : S1(read);
5923 src2 : S2(read);
5924 dst : S5(write);
5925 INS01 : ISS;
5926 NEON_FP : S5;
5927 %}
5928
5929 pipe_class fp_uop_s(vRegF dst, vRegF src)
5930 %{
5931 single_instruction;
5932 src : S1(read);
5933 dst : S5(write);
5934 INS01 : ISS;
5935 NEON_FP : S5;
5936 %}
5937
5938 pipe_class fp_uop_d(vRegD dst, vRegD src)
5939 %{
5940 single_instruction;
5941 src : S1(read);
5942 dst : S5(write);
5943 INS01 : ISS;
5944 NEON_FP : S5;
5945 %}
5946
5947 pipe_class fp_d2f(vRegF dst, vRegD src)
5948 %{
5949 single_instruction;
5950 src : S1(read);
5951 dst : S5(write);
5952 INS01 : ISS;
5953 NEON_FP : S5;
5954 %}
5955
5956 pipe_class fp_f2d(vRegD dst, vRegF src)
5957 %{
5958 single_instruction;
5959 src : S1(read);
5960 dst : S5(write);
5961 INS01 : ISS;
5962 NEON_FP : S5;
5963 %}
5964
5965 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5966 %{
5967 single_instruction;
5968 src : S1(read);
5969 dst : S5(write);
5970 INS01 : ISS;
5971 NEON_FP : S5;
5972 %}
5973
5974 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5975 %{
5976 single_instruction;
5977 src : S1(read);
5978 dst : S5(write);
5979 INS01 : ISS;
5980 NEON_FP : S5;
5981 %}
5982
5983 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5984 %{
5985 single_instruction;
5986 src : S1(read);
5987 dst : S5(write);
5988 INS01 : ISS;
5989 NEON_FP : S5;
5990 %}
5991
5992 pipe_class fp_l2f(vRegF dst, iRegL src)
5993 %{
5994 single_instruction;
5995 src : S1(read);
5996 dst : S5(write);
5997 INS01 : ISS;
5998 NEON_FP : S5;
5999 %}
6000
6001 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6002 %{
6003 single_instruction;
6004 src : S1(read);
6005 dst : S5(write);
6006 INS01 : ISS;
6007 NEON_FP : S5;
6008 %}
6009
6010 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6011 %{
6012 single_instruction;
6013 src : S1(read);
6014 dst : S5(write);
6015 INS01 : ISS;
6016 NEON_FP : S5;
6017 %}
6018
6019 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6020 %{
6021 single_instruction;
6022 src : S1(read);
6023 dst : S5(write);
6024 INS01 : ISS;
6025 NEON_FP : S5;
6026 %}
6027
6028 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6029 %{
6030 single_instruction;
6031 src : S1(read);
6032 dst : S5(write);
6033 INS01 : ISS;
6034 NEON_FP : S5;
6035 %}
6036
6037 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6038 %{
6039 single_instruction;
6040 src1 : S1(read);
6041 src2 : S2(read);
6042 dst : S5(write);
6043 INS0 : ISS;
6044 NEON_FP : S5;
6045 %}
6046
6047 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6048 %{
6049 single_instruction;
6050 src1 : S1(read);
6051 src2 : S2(read);
6052 dst : S5(write);
6053 INS0 : ISS;
6054 NEON_FP : S5;
6055 %}
6056
6057 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6058 %{
6059 single_instruction;
6060 cr : S1(read);
6061 src1 : S1(read);
6062 src2 : S1(read);
6063 dst : S3(write);
6064 INS01 : ISS;
6065 NEON_FP : S3;
6066 %}
6067
6068 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6069 %{
6070 single_instruction;
6071 cr : S1(read);
6072 src1 : S1(read);
6073 src2 : S1(read);
6074 dst : S3(write);
6075 INS01 : ISS;
6076 NEON_FP : S3;
6077 %}
6078
6079 pipe_class fp_imm_s(vRegF dst)
6080 %{
6081 single_instruction;
6082 dst : S3(write);
6083 INS01 : ISS;
6084 NEON_FP : S3;
6085 %}
6086
6087 pipe_class fp_imm_d(vRegD dst)
6088 %{
6089 single_instruction;
6090 dst : S3(write);
6091 INS01 : ISS;
6092 NEON_FP : S3;
6093 %}
6094
6095 pipe_class fp_load_constant_s(vRegF dst)
6096 %{
6097 single_instruction;
6098 dst : S4(write);
6099 INS01 : ISS;
6100 NEON_FP : S4;
6101 %}
6102
6103 pipe_class fp_load_constant_d(vRegD dst)
6104 %{
6105 single_instruction;
6106 dst : S4(write);
6107 INS01 : ISS;
6108 NEON_FP : S4;
6109 %}
6110
6111 //------- Integer ALU operations --------------------------
6112
6113 // Integer ALU reg-reg operation
6114 // Operands needed in EX1, result generated in EX2
6115 // Eg. ADD x0, x1, x2
6116 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6117 %{
6118 single_instruction;
6119 dst : EX2(write);
6120 src1 : EX1(read);
6121 src2 : EX1(read);
6122 INS01 : ISS; // Dual issue as instruction 0 or 1
6123 ALU : EX2;
6124 %}
6125
6126 // Integer ALU reg-reg operation with constant shift
6127 // Shifted register must be available in LATE_ISS instead of EX1
6128 // Eg. ADD x0, x1, x2, LSL #2
6129 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6130 %{
6131 single_instruction;
6132 dst : EX2(write);
6133 src1 : EX1(read);
6134 src2 : ISS(read);
6135 INS01 : ISS;
6136 ALU : EX2;
6137 %}
6138
6139 // Integer ALU reg operation with constant shift
6140 // Eg. LSL x0, x1, #shift
6141 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6142 %{
6143 single_instruction;
6144 dst : EX2(write);
6145 src1 : ISS(read);
6146 INS01 : ISS;
6147 ALU : EX2;
6148 %}
6149
6150 // Integer ALU reg-reg operation with variable shift
6151 // Both operands must be available in LATE_ISS instead of EX1
6152 // Result is available in EX1 instead of EX2
6153 // Eg. LSLV x0, x1, x2
6154 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6155 %{
6156 single_instruction;
6157 dst : EX1(write);
6158 src1 : ISS(read);
6159 src2 : ISS(read);
6160 INS01 : ISS;
6161 ALU : EX1;
6162 %}
6163
6164 // Integer ALU reg-reg operation with extract
6165 // As for _vshift above, but result generated in EX2
6166 // Eg. EXTR x0, x1, x2, #N
6167 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6168 %{
6169 single_instruction;
6170 dst : EX2(write);
6171 src1 : ISS(read);
6172 src2 : ISS(read);
6173 INS1 : ISS; // Can only dual issue as Instruction 1
6174 ALU : EX1;
6175 %}
6176
6177 // Integer ALU reg operation
6178 // Eg. NEG x0, x1
6179 pipe_class ialu_reg(iRegI dst, iRegI src)
6180 %{
6181 single_instruction;
6182 dst : EX2(write);
6183 src : EX1(read);
6184 INS01 : ISS;
6185 ALU : EX2;
6186 %}
6187
6188 // Integer ALU reg mmediate operation
6189 // Eg. ADD x0, x1, #N
6190 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6191 %{
6192 single_instruction;
6193 dst : EX2(write);
6194 src1 : EX1(read);
6195 INS01 : ISS;
6196 ALU : EX2;
6197 %}
6198
6199 // Integer ALU immediate operation (no source operands)
6200 // Eg. MOV x0, #N
6201 pipe_class ialu_imm(iRegI dst)
6202 %{
6203 single_instruction;
6204 dst : EX1(write);
6205 INS01 : ISS;
6206 ALU : EX1;
6207 %}
6208
6209 //------- Compare operation -------------------------------
6210
6211 // Compare reg-reg
6212 // Eg. CMP x0, x1
6213 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6214 %{
6215 single_instruction;
6216 // fixed_latency(16);
6217 cr : EX2(write);
6218 op1 : EX1(read);
6219 op2 : EX1(read);
6220 INS01 : ISS;
6221 ALU : EX2;
6222 %}
6223
6224 // Compare reg-reg
6225 // Eg. CMP x0, #N
6226 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6227 %{
6228 single_instruction;
6229 // fixed_latency(16);
6230 cr : EX2(write);
6231 op1 : EX1(read);
6232 INS01 : ISS;
6233 ALU : EX2;
6234 %}
6235
6236 //------- Conditional instructions ------------------------
6237
6238 // Conditional no operands
6239 // Eg. CSINC x0, zr, zr, <cond>
6240 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6241 %{
6242 single_instruction;
6243 cr : EX1(read);
6244 dst : EX2(write);
6245 INS01 : ISS;
6246 ALU : EX2;
6247 %}
6248
6249 // Conditional 2 operand
6250 // EG. CSEL X0, X1, X2, <cond>
6251 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6252 %{
6253 single_instruction;
6254 cr : EX1(read);
6255 src1 : EX1(read);
6256 src2 : EX1(read);
6257 dst : EX2(write);
6258 INS01 : ISS;
6259 ALU : EX2;
6260 %}
6261
6262 // Conditional 2 operand
6263 // EG. CSEL X0, X1, X2, <cond>
6264 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6265 %{
6266 single_instruction;
6267 cr : EX1(read);
6268 src : EX1(read);
6269 dst : EX2(write);
6270 INS01 : ISS;
6271 ALU : EX2;
6272 %}
6273
6274 //------- Multiply pipeline operations --------------------
6275
6276 // Multiply reg-reg
6277 // Eg. MUL w0, w1, w2
6278 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6279 %{
6280 single_instruction;
6281 dst : WR(write);
6282 src1 : ISS(read);
6283 src2 : ISS(read);
6284 INS01 : ISS;
6285 MAC : WR;
6286 %}
6287
6288 // Multiply accumulate
6289 // Eg. MADD w0, w1, w2, w3
6290 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6291 %{
6292 single_instruction;
6293 dst : WR(write);
6294 src1 : ISS(read);
6295 src2 : ISS(read);
6296 src3 : ISS(read);
6297 INS01 : ISS;
6298 MAC : WR;
6299 %}
6300
6301 // Eg. MUL w0, w1, w2
6302 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6303 %{
6304 single_instruction;
6305 fixed_latency(3); // Maximum latency for 64 bit mul
6306 dst : WR(write);
6307 src1 : ISS(read);
6308 src2 : ISS(read);
6309 INS01 : ISS;
6310 MAC : WR;
6311 %}
6312
6313 // Multiply accumulate
6314 // Eg. MADD w0, w1, w2, w3
6315 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6316 %{
6317 single_instruction;
6318 fixed_latency(3); // Maximum latency for 64 bit mul
6319 dst : WR(write);
6320 src1 : ISS(read);
6321 src2 : ISS(read);
6322 src3 : ISS(read);
6323 INS01 : ISS;
6324 MAC : WR;
6325 %}
6326
6327 //------- Divide pipeline operations --------------------
6328
6329 // Eg. SDIV w0, w1, w2
6330 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6331 %{
6332 single_instruction;
6333 fixed_latency(8); // Maximum latency for 32 bit divide
6334 dst : WR(write);
6335 src1 : ISS(read);
6336 src2 : ISS(read);
6337 INS0 : ISS; // Can only dual issue as instruction 0
6338 DIV : WR;
6339 %}
6340
6341 // Eg. SDIV x0, x1, x2
6342 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6343 %{
6344 single_instruction;
6345 fixed_latency(16); // Maximum latency for 64 bit divide
6346 dst : WR(write);
6347 src1 : ISS(read);
6348 src2 : ISS(read);
6349 INS0 : ISS; // Can only dual issue as instruction 0
6350 DIV : WR;
6351 %}
6352
6353 //------- Load pipeline operations ------------------------
6354
6355 // Load - prefetch
6356 // Eg. PFRM <mem>
6357 pipe_class iload_prefetch(memory mem)
6358 %{
6359 single_instruction;
6360 mem : ISS(read);
6361 INS01 : ISS;
6362 LDST : WR;
6363 %}
6364
6365 // Load - reg, mem
6366 // Eg. LDR x0, <mem>
6367 pipe_class iload_reg_mem(iRegI dst, memory mem)
6368 %{
6369 single_instruction;
6370 dst : WR(write);
6371 mem : ISS(read);
6372 INS01 : ISS;
6373 LDST : WR;
6374 %}
6375
6376 // Load - reg, reg
6377 // Eg. LDR x0, [sp, x1]
6378 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6379 %{
6380 single_instruction;
6381 dst : WR(write);
6382 src : ISS(read);
6383 INS01 : ISS;
6384 LDST : WR;
6385 %}
6386
6387 //------- Store pipeline operations -----------------------
6388
6389 // Store - zr, mem
6390 // Eg. STR zr, <mem>
6391 pipe_class istore_mem(memory mem)
6392 %{
6393 single_instruction;
6394 mem : ISS(read);
6395 INS01 : ISS;
6396 LDST : WR;
6397 %}
6398
6399 // Store - reg, mem
6400 // Eg. STR x0, <mem>
6401 pipe_class istore_reg_mem(iRegI src, memory mem)
6402 %{
6403 single_instruction;
6404 mem : ISS(read);
6405 src : EX2(read);
6406 INS01 : ISS;
6407 LDST : WR;
6408 %}
6409
6410 // Store - reg, reg
6411 // Eg. STR x0, [sp, x1]
6412 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6413 %{
6414 single_instruction;
6415 dst : ISS(read);
6416 src : EX2(read);
6417 INS01 : ISS;
6418 LDST : WR;
6419 %}
6420
6421 //------- Store pipeline operations -----------------------
6422
6423 // Branch
6424 pipe_class pipe_branch()
6425 %{
6426 single_instruction;
6427 INS01 : ISS;
6428 BRANCH : EX1;
6429 %}
6430
6431 // Conditional branch
6432 pipe_class pipe_branch_cond(rFlagsReg cr)
6433 %{
6434 single_instruction;
6435 cr : EX1(read);
6436 INS01 : ISS;
6437 BRANCH : EX1;
6438 %}
6439
6440 // Compare & Branch
6441 // EG. CBZ/CBNZ
6442 pipe_class pipe_cmp_branch(iRegI op1)
6443 %{
6444 single_instruction;
6445 op1 : EX1(read);
6446 INS01 : ISS;
6447 BRANCH : EX1;
6448 %}
6449
6450 //------- Synchronisation operations ----------------------
6451
6452 // Any operation requiring serialization.
6453 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6454 pipe_class pipe_serial()
6455 %{
6456 single_instruction;
6457 force_serialization;
6458 fixed_latency(16);
6459 INS01 : ISS(2); // Cannot dual issue with any other instruction
6460 LDST : WR;
6461 %}
6462
6463 // Generic big/slow expanded idiom - also serialized
6464 pipe_class pipe_slow()
6465 %{
6466 instruction_count(10);
6467 multiple_bundles;
6468 force_serialization;
6469 fixed_latency(16);
6470 INS01 : ISS(2); // Cannot dual issue with any other instruction
6471 LDST : WR;
6472 %}
6473
6474 // Empty pipeline class
6475 pipe_class pipe_class_empty()
6476 %{
6477 single_instruction;
6478 fixed_latency(0);
6479 %}
6480
6481 // Default pipeline class.
6482 pipe_class pipe_class_default()
6483 %{
6484 single_instruction;
6485 fixed_latency(2);
6486 %}
6487
6488 // Pipeline class for compares.
6489 pipe_class pipe_class_compare()
6490 %{
6491 single_instruction;
6492 fixed_latency(16);
6493 %}
6494
6495 // Pipeline class for memory operations.
6496 pipe_class pipe_class_memory()
6497 %{
6498 single_instruction;
6499 fixed_latency(16);
6500 %}
6501
6502 // Pipeline class for call.
6503 pipe_class pipe_class_call()
6504 %{
6505 single_instruction;
6506 fixed_latency(100);
6507 %}
6508
6509 // Define the class for the Nop node.
6510 define %{
6511 MachNop = pipe_class_empty;
6512 %}
6513
6514 %}
6515 //----------INSTRUCTIONS-------------------------------------------------------
6516 //
6517 // match -- States which machine-independent subtree may be replaced
6518 // by this instruction.
6519 // ins_cost -- The estimated cost of this instruction is used by instruction
6520 // selection to identify a minimum cost tree of machine
6521 // instructions that matches a tree of machine-independent
6522 // instructions.
6523 // format -- A string providing the disassembly for this instruction.
6524 // The value of an instruction's operand may be inserted
6525 // by referring to it with a '$' prefix.
6526 // opcode -- Three instruction opcodes may be provided. These are referred
6527 // to within an encode class as $primary, $secondary, and $tertiary
6528 // rrspectively. The primary opcode is commonly used to
6529 // indicate the type of machine instruction, while secondary
6530 // and tertiary are often used for prefix options or addressing
6531 // modes.
6532 // ins_encode -- A list of encode classes with parameters. The encode class
6533 // name must have been defined in an 'enc_class' specification
6534 // in the encode section of the architecture description.
6535
6536 // ============================================================================
6537 // Memory (Load/Store) Instructions
6538
6539 // Load Instructions
6540
6541 // Load Byte (8 bit signed)
6542 instruct loadB(iRegINoSp dst, memory1 mem)
6543 %{
6544 match(Set dst (LoadB mem));
6545 predicate(!needs_acquiring_load(n));
6546
6547 ins_cost(4 * INSN_COST);
6548 format %{ "ldrsbw $dst, $mem\t# byte" %}
6549
6550 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6551
6552 ins_pipe(iload_reg_mem);
6553 %}
6554
6555 // Load Byte (8 bit signed) into long
6556 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6557 %{
6558 match(Set dst (ConvI2L (LoadB mem)));
6559 predicate(!needs_acquiring_load(n->in(1)));
6560
6561 ins_cost(4 * INSN_COST);
6562 format %{ "ldrsb $dst, $mem\t# byte" %}
6563
6564 ins_encode(aarch64_enc_ldrsb(dst, mem));
6565
6566 ins_pipe(iload_reg_mem);
6567 %}
6568
6569 // Load Byte (8 bit unsigned)
6570 instruct loadUB(iRegINoSp dst, memory1 mem)
6571 %{
6572 match(Set dst (LoadUB mem));
6573 predicate(!needs_acquiring_load(n));
6574
6575 ins_cost(4 * INSN_COST);
6576 format %{ "ldrbw $dst, $mem\t# byte" %}
6577
6578 ins_encode(aarch64_enc_ldrb(dst, mem));
6579
6580 ins_pipe(iload_reg_mem);
6581 %}
6582
6583 // Load Byte (8 bit unsigned) into long
6584 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6585 %{
6586 match(Set dst (ConvI2L (LoadUB mem)));
6587 predicate(!needs_acquiring_load(n->in(1)));
6588
6589 ins_cost(4 * INSN_COST);
6590 format %{ "ldrb $dst, $mem\t# byte" %}
6591
6592 ins_encode(aarch64_enc_ldrb(dst, mem));
6593
6594 ins_pipe(iload_reg_mem);
6595 %}
6596
6597 // Load Short (16 bit signed)
6598 instruct loadS(iRegINoSp dst, memory2 mem)
6599 %{
6600 match(Set dst (LoadS mem));
6601 predicate(!needs_acquiring_load(n));
6602
6603 ins_cost(4 * INSN_COST);
6604 format %{ "ldrshw $dst, $mem\t# short" %}
6605
6606 ins_encode(aarch64_enc_ldrshw(dst, mem));
6607
6608 ins_pipe(iload_reg_mem);
6609 %}
6610
6611 // Load Short (16 bit signed) into long
6612 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6613 %{
6614 match(Set dst (ConvI2L (LoadS mem)));
6615 predicate(!needs_acquiring_load(n->in(1)));
6616
6617 ins_cost(4 * INSN_COST);
6618 format %{ "ldrsh $dst, $mem\t# short" %}
6619
6620 ins_encode(aarch64_enc_ldrsh(dst, mem));
6621
6622 ins_pipe(iload_reg_mem);
6623 %}
6624
6625 // Load Char (16 bit unsigned)
6626 instruct loadUS(iRegINoSp dst, memory2 mem)
6627 %{
6628 match(Set dst (LoadUS mem));
6629 predicate(!needs_acquiring_load(n));
6630
6631 ins_cost(4 * INSN_COST);
6632 format %{ "ldrh $dst, $mem\t# short" %}
6633
6634 ins_encode(aarch64_enc_ldrh(dst, mem));
6635
6636 ins_pipe(iload_reg_mem);
6637 %}
6638
6639 // Load Short/Char (16 bit unsigned) into long
6640 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6641 %{
6642 match(Set dst (ConvI2L (LoadUS mem)));
6643 predicate(!needs_acquiring_load(n->in(1)));
6644
6645 ins_cost(4 * INSN_COST);
6646 format %{ "ldrh $dst, $mem\t# short" %}
6647
6648 ins_encode(aarch64_enc_ldrh(dst, mem));
6649
6650 ins_pipe(iload_reg_mem);
6651 %}
6652
6653 // Load Integer (32 bit signed)
6654 instruct loadI(iRegINoSp dst, memory4 mem)
6655 %{
6656 match(Set dst (LoadI mem));
6657 predicate(!needs_acquiring_load(n));
6658
6659 ins_cost(4 * INSN_COST);
6660 format %{ "ldrw $dst, $mem\t# int" %}
6661
6662 ins_encode(aarch64_enc_ldrw(dst, mem));
6663
6664 ins_pipe(iload_reg_mem);
6665 %}
6666
6667 // Load Integer (32 bit signed) into long
6668 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6669 %{
6670 match(Set dst (ConvI2L (LoadI mem)));
6671 predicate(!needs_acquiring_load(n->in(1)));
6672
6673 ins_cost(4 * INSN_COST);
6674 format %{ "ldrsw $dst, $mem\t# int" %}
6675
6676 ins_encode(aarch64_enc_ldrsw(dst, mem));
6677
6678 ins_pipe(iload_reg_mem);
6679 %}
6680
6681 // Load Integer (32 bit unsigned) into long
6682 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6683 %{
6684 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6685 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6686
6687 ins_cost(4 * INSN_COST);
6688 format %{ "ldrw $dst, $mem\t# int" %}
6689
6690 ins_encode(aarch64_enc_ldrw(dst, mem));
6691
6692 ins_pipe(iload_reg_mem);
6693 %}
6694
6695 // Load Long (64 bit signed)
6696 instruct loadL(iRegLNoSp dst, memory8 mem)
6697 %{
6698 match(Set dst (LoadL mem));
6699 predicate(!needs_acquiring_load(n));
6700
6701 ins_cost(4 * INSN_COST);
6702 format %{ "ldr $dst, $mem\t# int" %}
6703
6704 ins_encode(aarch64_enc_ldr(dst, mem));
6705
6706 ins_pipe(iload_reg_mem);
6707 %}
6708
6709 // Load Range
6710 instruct loadRange(iRegINoSp dst, memory4 mem)
6711 %{
6712 match(Set dst (LoadRange mem));
6713
6714 ins_cost(4 * INSN_COST);
6715 format %{ "ldrw $dst, $mem\t# range" %}
6716
6717 ins_encode(aarch64_enc_ldrw(dst, mem));
6718
6719 ins_pipe(iload_reg_mem);
6720 %}
6721
6722 // Load Pointer
6723 instruct loadP(iRegPNoSp dst, memory8 mem)
6724 %{
6725 match(Set dst (LoadP mem));
6726 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6727
6728 ins_cost(4 * INSN_COST);
6729 format %{ "ldr $dst, $mem\t# ptr" %}
6730
6731 ins_encode(aarch64_enc_ldr(dst, mem));
6732
6733 ins_pipe(iload_reg_mem);
6734 %}
6735
6736 // Load Compressed Pointer
6737 instruct loadN(iRegNNoSp dst, memory4 mem)
6738 %{
6739 match(Set dst (LoadN mem));
6740 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6741
6742 ins_cost(4 * INSN_COST);
6743 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6744
6745 ins_encode(aarch64_enc_ldrw(dst, mem));
6746
6747 ins_pipe(iload_reg_mem);
6748 %}
6749
6750 // Load Klass Pointer
6751 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6752 %{
6753 match(Set dst (LoadKlass mem));
6754 predicate(!needs_acquiring_load(n));
6755
6756 ins_cost(4 * INSN_COST);
6757 format %{ "ldr $dst, $mem\t# class" %}
6758
6759 ins_encode(aarch64_enc_ldr(dst, mem));
6760
6761 ins_pipe(iload_reg_mem);
6762 %}
6763
6764 // Load Narrow Klass Pointer
6765 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6766 %{
6767 match(Set dst (LoadNKlass mem));
6768 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6769
6770 ins_cost(4 * INSN_COST);
6771 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6772
6773 ins_encode(aarch64_enc_ldrw(dst, mem));
6774
6775 ins_pipe(iload_reg_mem);
6776 %}
6777
6778 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory_noindex mem)
6779 %{
6780 match(Set dst (LoadNKlass mem));
6781 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6782
6783 ins_cost(4 * INSN_COST);
6784 format %{
6785 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6786 "lsrw $dst, $dst, markWord::klass_shift"
6787 %}
6788 ins_encode %{
6789 assert($mem$$index$$Register == noreg, "must not have indexed address");
6790 // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
6791 // obj-start, so that we can load from the object's mark-word instead.
6792 __ ldrw($dst$$Register, Address($mem$$base$$Register, $mem$$disp - Type::klass_offset()));
6793 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift);
6794 %}
6795 ins_pipe(iload_reg_mem);
6796 %}
6797
6798 // Load Float
6799 instruct loadF(vRegF dst, memory4 mem)
6800 %{
6801 match(Set dst (LoadF mem));
6802 predicate(!needs_acquiring_load(n));
6803
6804 ins_cost(4 * INSN_COST);
6805 format %{ "ldrs $dst, $mem\t# float" %}
6806
6807 ins_encode( aarch64_enc_ldrs(dst, mem) );
6808
6809 ins_pipe(pipe_class_memory);
6810 %}
6811
6812 // Load Double
6813 instruct loadD(vRegD dst, memory8 mem)
6814 %{
6815 match(Set dst (LoadD mem));
6816 predicate(!needs_acquiring_load(n));
6817
6818 ins_cost(4 * INSN_COST);
6819 format %{ "ldrd $dst, $mem\t# double" %}
6820
6821 ins_encode( aarch64_enc_ldrd(dst, mem) );
6822
6823 ins_pipe(pipe_class_memory);
6824 %}
6825
6826
6827 // Load Int Constant
6828 instruct loadConI(iRegINoSp dst, immI src)
6829 %{
6830 match(Set dst src);
6831
6832 ins_cost(INSN_COST);
6833 format %{ "mov $dst, $src\t# int" %}
6834
6835 ins_encode( aarch64_enc_movw_imm(dst, src) );
6836
6837 ins_pipe(ialu_imm);
6838 %}
6839
6840 // Load Long Constant
6841 instruct loadConL(iRegLNoSp dst, immL src)
6842 %{
6843 match(Set dst src);
6844
6845 ins_cost(INSN_COST);
6846 format %{ "mov $dst, $src\t# long" %}
6847
6848 ins_encode( aarch64_enc_mov_imm(dst, src) );
6849
6850 ins_pipe(ialu_imm);
6851 %}
6852
6853 // Load Pointer Constant
6854
6855 instruct loadConP(iRegPNoSp dst, immP con)
6856 %{
6857 match(Set dst con);
6858
6859 ins_cost(INSN_COST * 4);
6860 format %{
6861 "mov $dst, $con\t# ptr\n\t"
6862 %}
6863
6864 ins_encode(aarch64_enc_mov_p(dst, con));
6865
6866 ins_pipe(ialu_imm);
6867 %}
6868
6869 // Load Null Pointer Constant
6870
6871 instruct loadConP0(iRegPNoSp dst, immP0 con)
6872 %{
6873 match(Set dst con);
6874
6875 ins_cost(INSN_COST);
6876 format %{ "mov $dst, $con\t# nullptr ptr" %}
6877
6878 ins_encode(aarch64_enc_mov_p0(dst, con));
6879
6880 ins_pipe(ialu_imm);
6881 %}
6882
6883 // Load Pointer Constant One
6884
6885 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6886 %{
6887 match(Set dst con);
6888
6889 ins_cost(INSN_COST);
6890 format %{ "mov $dst, $con\t# nullptr ptr" %}
6891
6892 ins_encode(aarch64_enc_mov_p1(dst, con));
6893
6894 ins_pipe(ialu_imm);
6895 %}
6896
6897 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6898 %{
6899 match(Set dst con);
6900
6901 ins_cost(INSN_COST);
6902 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6903
6904 ins_encode %{
6905 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6906 %}
6907
6908 ins_pipe(ialu_imm);
6909 %}
6910
6911 // Load Narrow Pointer Constant
6912
6913 instruct loadConN(iRegNNoSp dst, immN con)
6914 %{
6915 match(Set dst con);
6916
6917 ins_cost(INSN_COST * 4);
6918 format %{ "mov $dst, $con\t# compressed ptr" %}
6919
6920 ins_encode(aarch64_enc_mov_n(dst, con));
6921
6922 ins_pipe(ialu_imm);
6923 %}
6924
6925 // Load Narrow Null Pointer Constant
6926
6927 instruct loadConN0(iRegNNoSp dst, immN0 con)
6928 %{
6929 match(Set dst con);
6930
6931 ins_cost(INSN_COST);
6932 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6933
6934 ins_encode(aarch64_enc_mov_n0(dst, con));
6935
6936 ins_pipe(ialu_imm);
6937 %}
6938
6939 // Load Narrow Klass Constant
6940
6941 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6942 %{
6943 match(Set dst con);
6944
6945 ins_cost(INSN_COST);
6946 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6947
6948 ins_encode(aarch64_enc_mov_nk(dst, con));
6949
6950 ins_pipe(ialu_imm);
6951 %}
6952
6953 // Load Packed Float Constant
6954
6955 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6956 match(Set dst con);
6957 ins_cost(INSN_COST * 4);
6958 format %{ "fmovs $dst, $con"%}
6959 ins_encode %{
6960 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6961 %}
6962
6963 ins_pipe(fp_imm_s);
6964 %}
6965
6966 // Load Float Constant
6967
6968 instruct loadConF(vRegF dst, immF con) %{
6969 match(Set dst con);
6970
6971 ins_cost(INSN_COST * 4);
6972
6973 format %{
6974 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6975 %}
6976
6977 ins_encode %{
6978 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6979 %}
6980
6981 ins_pipe(fp_load_constant_s);
6982 %}
6983
6984 // Load Packed Double Constant
6985
6986 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6987 match(Set dst con);
6988 ins_cost(INSN_COST);
6989 format %{ "fmovd $dst, $con"%}
6990 ins_encode %{
6991 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6992 %}
6993
6994 ins_pipe(fp_imm_d);
6995 %}
6996
6997 // Load Double Constant
6998
6999 instruct loadConD(vRegD dst, immD con) %{
7000 match(Set dst con);
7001
7002 ins_cost(INSN_COST * 5);
7003 format %{
7004 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7005 %}
7006
7007 ins_encode %{
7008 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7009 %}
7010
7011 ins_pipe(fp_load_constant_d);
7012 %}
7013
7014 // Load Half Float Constant
7015 instruct loadConH(vRegF dst, immH con) %{
7016 match(Set dst con);
7017 format %{ "mov rscratch1, $con\n\t"
7018 "fmov $dst, rscratch1"
7019 %}
7020 ins_encode %{
7021 __ movw(rscratch1, (uint32_t)$con$$constant);
7022 __ fmovs($dst$$FloatRegister, rscratch1);
7023 %}
7024 ins_pipe(pipe_class_default);
7025 %}
7026
7027 // Store Instructions
7028
7029 // Store Byte
7030 instruct storeB(iRegIorL2I src, memory1 mem)
7031 %{
7032 match(Set mem (StoreB mem src));
7033 predicate(!needs_releasing_store(n));
7034
7035 ins_cost(INSN_COST);
7036 format %{ "strb $src, $mem\t# byte" %}
7037
7038 ins_encode(aarch64_enc_strb(src, mem));
7039
7040 ins_pipe(istore_reg_mem);
7041 %}
7042
7043
7044 instruct storeimmB0(immI0 zero, memory1 mem)
7045 %{
7046 match(Set mem (StoreB mem zero));
7047 predicate(!needs_releasing_store(n));
7048
7049 ins_cost(INSN_COST);
7050 format %{ "strb rscractch2, $mem\t# byte" %}
7051
7052 ins_encode(aarch64_enc_strb0(mem));
7053
7054 ins_pipe(istore_mem);
7055 %}
7056
7057 // Store Char/Short
7058 instruct storeC(iRegIorL2I src, memory2 mem)
7059 %{
7060 match(Set mem (StoreC mem src));
7061 predicate(!needs_releasing_store(n));
7062
7063 ins_cost(INSN_COST);
7064 format %{ "strh $src, $mem\t# short" %}
7065
7066 ins_encode(aarch64_enc_strh(src, mem));
7067
7068 ins_pipe(istore_reg_mem);
7069 %}
7070
7071 instruct storeimmC0(immI0 zero, memory2 mem)
7072 %{
7073 match(Set mem (StoreC mem zero));
7074 predicate(!needs_releasing_store(n));
7075
7076 ins_cost(INSN_COST);
7077 format %{ "strh zr, $mem\t# short" %}
7078
7079 ins_encode(aarch64_enc_strh0(mem));
7080
7081 ins_pipe(istore_mem);
7082 %}
7083
7084 // Store Integer
7085
7086 instruct storeI(iRegIorL2I src, memory4 mem)
7087 %{
7088 match(Set mem(StoreI mem src));
7089 predicate(!needs_releasing_store(n));
7090
7091 ins_cost(INSN_COST);
7092 format %{ "strw $src, $mem\t# int" %}
7093
7094 ins_encode(aarch64_enc_strw(src, mem));
7095
7096 ins_pipe(istore_reg_mem);
7097 %}
7098
7099 instruct storeimmI0(immI0 zero, memory4 mem)
7100 %{
7101 match(Set mem(StoreI mem zero));
7102 predicate(!needs_releasing_store(n));
7103
7104 ins_cost(INSN_COST);
7105 format %{ "strw zr, $mem\t# int" %}
7106
7107 ins_encode(aarch64_enc_strw0(mem));
7108
7109 ins_pipe(istore_mem);
7110 %}
7111
7112 // Store Long (64 bit signed)
7113 instruct storeL(iRegL src, memory8 mem)
7114 %{
7115 match(Set mem (StoreL mem src));
7116 predicate(!needs_releasing_store(n));
7117
7118 ins_cost(INSN_COST);
7119 format %{ "str $src, $mem\t# int" %}
7120
7121 ins_encode(aarch64_enc_str(src, mem));
7122
7123 ins_pipe(istore_reg_mem);
7124 %}
7125
7126 // Store Long (64 bit signed)
7127 instruct storeimmL0(immL0 zero, memory8 mem)
7128 %{
7129 match(Set mem (StoreL mem zero));
7130 predicate(!needs_releasing_store(n));
7131
7132 ins_cost(INSN_COST);
7133 format %{ "str zr, $mem\t# int" %}
7134
7135 ins_encode(aarch64_enc_str0(mem));
7136
7137 ins_pipe(istore_mem);
7138 %}
7139
7140 // Store Pointer
7141 instruct storeP(iRegP src, memory8 mem)
7142 %{
7143 match(Set mem (StoreP mem src));
7144 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7145
7146 ins_cost(INSN_COST);
7147 format %{ "str $src, $mem\t# ptr" %}
7148
7149 ins_encode(aarch64_enc_str(src, mem));
7150
7151 ins_pipe(istore_reg_mem);
7152 %}
7153
7154 // Store Pointer
7155 instruct storeimmP0(immP0 zero, memory8 mem)
7156 %{
7157 match(Set mem (StoreP mem zero));
7158 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7159
7160 ins_cost(INSN_COST);
7161 format %{ "str zr, $mem\t# ptr" %}
7162
7163 ins_encode(aarch64_enc_str0(mem));
7164
7165 ins_pipe(istore_mem);
7166 %}
7167
7168 // Store Compressed Pointer
7169 instruct storeN(iRegN src, memory4 mem)
7170 %{
7171 match(Set mem (StoreN mem src));
7172 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7173
7174 ins_cost(INSN_COST);
7175 format %{ "strw $src, $mem\t# compressed ptr" %}
7176
7177 ins_encode(aarch64_enc_strw(src, mem));
7178
7179 ins_pipe(istore_reg_mem);
7180 %}
7181
7182 instruct storeImmN0(immN0 zero, memory4 mem)
7183 %{
7184 match(Set mem (StoreN mem zero));
7185 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7186
7187 ins_cost(INSN_COST);
7188 format %{ "strw zr, $mem\t# compressed ptr" %}
7189
7190 ins_encode(aarch64_enc_strw0(mem));
7191
7192 ins_pipe(istore_mem);
7193 %}
7194
7195 // Store Float
7196 instruct storeF(vRegF src, memory4 mem)
7197 %{
7198 match(Set mem (StoreF mem src));
7199 predicate(!needs_releasing_store(n));
7200
7201 ins_cost(INSN_COST);
7202 format %{ "strs $src, $mem\t# float" %}
7203
7204 ins_encode( aarch64_enc_strs(src, mem) );
7205
7206 ins_pipe(pipe_class_memory);
7207 %}
7208
7209 // TODO
7210 // implement storeImmF0 and storeFImmPacked
7211
7212 // Store Double
7213 instruct storeD(vRegD src, memory8 mem)
7214 %{
7215 match(Set mem (StoreD mem src));
7216 predicate(!needs_releasing_store(n));
7217
7218 ins_cost(INSN_COST);
7219 format %{ "strd $src, $mem\t# double" %}
7220
7221 ins_encode( aarch64_enc_strd(src, mem) );
7222
7223 ins_pipe(pipe_class_memory);
7224 %}
7225
7226 // Store Compressed Klass Pointer
7227 instruct storeNKlass(iRegN src, memory4 mem)
7228 %{
7229 predicate(!needs_releasing_store(n));
7230 match(Set mem (StoreNKlass mem src));
7231
7232 ins_cost(INSN_COST);
7233 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7234
7235 ins_encode(aarch64_enc_strw(src, mem));
7236
7237 ins_pipe(istore_reg_mem);
7238 %}
7239
7240 // TODO
7241 // implement storeImmD0 and storeDImmPacked
7242
7243 // prefetch instructions
7244 // Must be safe to execute with invalid address (cannot fault).
7245
7246 instruct prefetchalloc( memory8 mem ) %{
7247 match(PrefetchAllocation mem);
7248
7249 ins_cost(INSN_COST);
7250 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7251
7252 ins_encode( aarch64_enc_prefetchw(mem) );
7253
7254 ins_pipe(iload_prefetch);
7255 %}
7256
7257 // ---------------- volatile loads and stores ----------------
7258
7259 // Load Byte (8 bit signed)
7260 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7261 %{
7262 match(Set dst (LoadB mem));
7263
7264 ins_cost(VOLATILE_REF_COST);
7265 format %{ "ldarsb $dst, $mem\t# byte" %}
7266
7267 ins_encode(aarch64_enc_ldarsb(dst, mem));
7268
7269 ins_pipe(pipe_serial);
7270 %}
7271
7272 // Load Byte (8 bit signed) into long
7273 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7274 %{
7275 match(Set dst (ConvI2L (LoadB mem)));
7276
7277 ins_cost(VOLATILE_REF_COST);
7278 format %{ "ldarsb $dst, $mem\t# byte" %}
7279
7280 ins_encode(aarch64_enc_ldarsb(dst, mem));
7281
7282 ins_pipe(pipe_serial);
7283 %}
7284
7285 // Load Byte (8 bit unsigned)
7286 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7287 %{
7288 match(Set dst (LoadUB mem));
7289
7290 ins_cost(VOLATILE_REF_COST);
7291 format %{ "ldarb $dst, $mem\t# byte" %}
7292
7293 ins_encode(aarch64_enc_ldarb(dst, mem));
7294
7295 ins_pipe(pipe_serial);
7296 %}
7297
7298 // Load Byte (8 bit unsigned) into long
7299 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7300 %{
7301 match(Set dst (ConvI2L (LoadUB mem)));
7302
7303 ins_cost(VOLATILE_REF_COST);
7304 format %{ "ldarb $dst, $mem\t# byte" %}
7305
7306 ins_encode(aarch64_enc_ldarb(dst, mem));
7307
7308 ins_pipe(pipe_serial);
7309 %}
7310
7311 // Load Short (16 bit signed)
7312 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7313 %{
7314 match(Set dst (LoadS mem));
7315
7316 ins_cost(VOLATILE_REF_COST);
7317 format %{ "ldarshw $dst, $mem\t# short" %}
7318
7319 ins_encode(aarch64_enc_ldarshw(dst, mem));
7320
7321 ins_pipe(pipe_serial);
7322 %}
7323
7324 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7325 %{
7326 match(Set dst (LoadUS mem));
7327
7328 ins_cost(VOLATILE_REF_COST);
7329 format %{ "ldarhw $dst, $mem\t# short" %}
7330
7331 ins_encode(aarch64_enc_ldarhw(dst, mem));
7332
7333 ins_pipe(pipe_serial);
7334 %}
7335
7336 // Load Short/Char (16 bit unsigned) into long
7337 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7338 %{
7339 match(Set dst (ConvI2L (LoadUS mem)));
7340
7341 ins_cost(VOLATILE_REF_COST);
7342 format %{ "ldarh $dst, $mem\t# short" %}
7343
7344 ins_encode(aarch64_enc_ldarh(dst, mem));
7345
7346 ins_pipe(pipe_serial);
7347 %}
7348
7349 // Load Short/Char (16 bit signed) into long
7350 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7351 %{
7352 match(Set dst (ConvI2L (LoadS mem)));
7353
7354 ins_cost(VOLATILE_REF_COST);
7355 format %{ "ldarh $dst, $mem\t# short" %}
7356
7357 ins_encode(aarch64_enc_ldarsh(dst, mem));
7358
7359 ins_pipe(pipe_serial);
7360 %}
7361
7362 // Load Integer (32 bit signed)
7363 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7364 %{
7365 match(Set dst (LoadI mem));
7366
7367 ins_cost(VOLATILE_REF_COST);
7368 format %{ "ldarw $dst, $mem\t# int" %}
7369
7370 ins_encode(aarch64_enc_ldarw(dst, mem));
7371
7372 ins_pipe(pipe_serial);
7373 %}
7374
7375 // Load Integer (32 bit unsigned) into long
7376 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7377 %{
7378 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7379
7380 ins_cost(VOLATILE_REF_COST);
7381 format %{ "ldarw $dst, $mem\t# int" %}
7382
7383 ins_encode(aarch64_enc_ldarw(dst, mem));
7384
7385 ins_pipe(pipe_serial);
7386 %}
7387
7388 // Load Long (64 bit signed)
7389 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7390 %{
7391 match(Set dst (LoadL mem));
7392
7393 ins_cost(VOLATILE_REF_COST);
7394 format %{ "ldar $dst, $mem\t# int" %}
7395
7396 ins_encode(aarch64_enc_ldar(dst, mem));
7397
7398 ins_pipe(pipe_serial);
7399 %}
7400
7401 // Load Pointer
7402 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7403 %{
7404 match(Set dst (LoadP mem));
7405 predicate(n->as_Load()->barrier_data() == 0);
7406
7407 ins_cost(VOLATILE_REF_COST);
7408 format %{ "ldar $dst, $mem\t# ptr" %}
7409
7410 ins_encode(aarch64_enc_ldar(dst, mem));
7411
7412 ins_pipe(pipe_serial);
7413 %}
7414
7415 // Load Compressed Pointer
7416 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7417 %{
7418 match(Set dst (LoadN mem));
7419 predicate(n->as_Load()->barrier_data() == 0);
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7423
7424 ins_encode(aarch64_enc_ldarw(dst, mem));
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Float
7430 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set dst (LoadF mem));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "ldars $dst, $mem\t# float" %}
7436
7437 ins_encode( aarch64_enc_fldars(dst, mem) );
7438
7439 ins_pipe(pipe_serial);
7440 %}
7441
7442 // Load Double
7443 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set dst (LoadD mem));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "ldard $dst, $mem\t# double" %}
7449
7450 ins_encode( aarch64_enc_fldard(dst, mem) );
7451
7452 ins_pipe(pipe_serial);
7453 %}
7454
7455 // Store Byte
7456 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7457 %{
7458 match(Set mem (StoreB mem src));
7459
7460 ins_cost(VOLATILE_REF_COST);
7461 format %{ "stlrb $src, $mem\t# byte" %}
7462
7463 ins_encode(aarch64_enc_stlrb(src, mem));
7464
7465 ins_pipe(pipe_class_memory);
7466 %}
7467
7468 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7469 %{
7470 match(Set mem (StoreB mem zero));
7471
7472 ins_cost(VOLATILE_REF_COST);
7473 format %{ "stlrb zr, $mem\t# byte" %}
7474
7475 ins_encode(aarch64_enc_stlrb0(mem));
7476
7477 ins_pipe(pipe_class_memory);
7478 %}
7479
7480 // Store Char/Short
7481 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7482 %{
7483 match(Set mem (StoreC mem src));
7484
7485 ins_cost(VOLATILE_REF_COST);
7486 format %{ "stlrh $src, $mem\t# short" %}
7487
7488 ins_encode(aarch64_enc_stlrh(src, mem));
7489
7490 ins_pipe(pipe_class_memory);
7491 %}
7492
7493 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7494 %{
7495 match(Set mem (StoreC mem zero));
7496
7497 ins_cost(VOLATILE_REF_COST);
7498 format %{ "stlrh zr, $mem\t# short" %}
7499
7500 ins_encode(aarch64_enc_stlrh0(mem));
7501
7502 ins_pipe(pipe_class_memory);
7503 %}
7504
7505 // Store Integer
7506
7507 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7508 %{
7509 match(Set mem(StoreI mem src));
7510
7511 ins_cost(VOLATILE_REF_COST);
7512 format %{ "stlrw $src, $mem\t# int" %}
7513
7514 ins_encode(aarch64_enc_stlrw(src, mem));
7515
7516 ins_pipe(pipe_class_memory);
7517 %}
7518
7519 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7520 %{
7521 match(Set mem(StoreI mem zero));
7522
7523 ins_cost(VOLATILE_REF_COST);
7524 format %{ "stlrw zr, $mem\t# int" %}
7525
7526 ins_encode(aarch64_enc_stlrw0(mem));
7527
7528 ins_pipe(pipe_class_memory);
7529 %}
7530
7531 // Store Long (64 bit signed)
7532 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7533 %{
7534 match(Set mem (StoreL mem src));
7535
7536 ins_cost(VOLATILE_REF_COST);
7537 format %{ "stlr $src, $mem\t# int" %}
7538
7539 ins_encode(aarch64_enc_stlr(src, mem));
7540
7541 ins_pipe(pipe_class_memory);
7542 %}
7543
7544 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7545 %{
7546 match(Set mem (StoreL mem zero));
7547
7548 ins_cost(VOLATILE_REF_COST);
7549 format %{ "stlr zr, $mem\t# int" %}
7550
7551 ins_encode(aarch64_enc_stlr0(mem));
7552
7553 ins_pipe(pipe_class_memory);
7554 %}
7555
7556 // Store Pointer
7557 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7558 %{
7559 match(Set mem (StoreP mem src));
7560 predicate(n->as_Store()->barrier_data() == 0);
7561
7562 ins_cost(VOLATILE_REF_COST);
7563 format %{ "stlr $src, $mem\t# ptr" %}
7564
7565 ins_encode(aarch64_enc_stlr(src, mem));
7566
7567 ins_pipe(pipe_class_memory);
7568 %}
7569
7570 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7571 %{
7572 match(Set mem (StoreP mem zero));
7573 predicate(n->as_Store()->barrier_data() == 0);
7574
7575 ins_cost(VOLATILE_REF_COST);
7576 format %{ "stlr zr, $mem\t# ptr" %}
7577
7578 ins_encode(aarch64_enc_stlr0(mem));
7579
7580 ins_pipe(pipe_class_memory);
7581 %}
7582
7583 // Store Compressed Pointer
7584 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7585 %{
7586 match(Set mem (StoreN mem src));
7587 predicate(n->as_Store()->barrier_data() == 0);
7588
7589 ins_cost(VOLATILE_REF_COST);
7590 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7591
7592 ins_encode(aarch64_enc_stlrw(src, mem));
7593
7594 ins_pipe(pipe_class_memory);
7595 %}
7596
7597 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7598 %{
7599 match(Set mem (StoreN mem zero));
7600 predicate(n->as_Store()->barrier_data() == 0);
7601
7602 ins_cost(VOLATILE_REF_COST);
7603 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7604
7605 ins_encode(aarch64_enc_stlrw0(mem));
7606
7607 ins_pipe(pipe_class_memory);
7608 %}
7609
7610 // Store Float
7611 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7612 %{
7613 match(Set mem (StoreF mem src));
7614
7615 ins_cost(VOLATILE_REF_COST);
7616 format %{ "stlrs $src, $mem\t# float" %}
7617
7618 ins_encode( aarch64_enc_fstlrs(src, mem) );
7619
7620 ins_pipe(pipe_class_memory);
7621 %}
7622
7623 // TODO
7624 // implement storeImmF0 and storeFImmPacked
7625
7626 // Store Double
7627 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7628 %{
7629 match(Set mem (StoreD mem src));
7630
7631 ins_cost(VOLATILE_REF_COST);
7632 format %{ "stlrd $src, $mem\t# double" %}
7633
7634 ins_encode( aarch64_enc_fstlrd(src, mem) );
7635
7636 ins_pipe(pipe_class_memory);
7637 %}
7638
7639 // ---------------- end of volatile loads and stores ----------------
7640
7641 instruct cacheWB(indirect addr)
7642 %{
7643 predicate(VM_Version::supports_data_cache_line_flush());
7644 match(CacheWB addr);
7645
7646 ins_cost(100);
7647 format %{"cache wb $addr" %}
7648 ins_encode %{
7649 assert($addr->index_position() < 0, "should be");
7650 assert($addr$$disp == 0, "should be");
7651 __ cache_wb(Address($addr$$base$$Register, 0));
7652 %}
7653 ins_pipe(pipe_slow); // XXX
7654 %}
7655
7656 instruct cacheWBPreSync()
7657 %{
7658 predicate(VM_Version::supports_data_cache_line_flush());
7659 match(CacheWBPreSync);
7660
7661 ins_cost(100);
7662 format %{"cache wb presync" %}
7663 ins_encode %{
7664 __ cache_wbsync(true);
7665 %}
7666 ins_pipe(pipe_slow); // XXX
7667 %}
7668
7669 instruct cacheWBPostSync()
7670 %{
7671 predicate(VM_Version::supports_data_cache_line_flush());
7672 match(CacheWBPostSync);
7673
7674 ins_cost(100);
7675 format %{"cache wb postsync" %}
7676 ins_encode %{
7677 __ cache_wbsync(false);
7678 %}
7679 ins_pipe(pipe_slow); // XXX
7680 %}
7681
7682 // ============================================================================
7683 // BSWAP Instructions
7684
7685 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7686 match(Set dst (ReverseBytesI src));
7687
7688 ins_cost(INSN_COST);
7689 format %{ "revw $dst, $src" %}
7690
7691 ins_encode %{
7692 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7693 %}
7694
7695 ins_pipe(ialu_reg);
7696 %}
7697
7698 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7699 match(Set dst (ReverseBytesL src));
7700
7701 ins_cost(INSN_COST);
7702 format %{ "rev $dst, $src" %}
7703
7704 ins_encode %{
7705 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7706 %}
7707
7708 ins_pipe(ialu_reg);
7709 %}
7710
7711 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7712 match(Set dst (ReverseBytesUS src));
7713
7714 ins_cost(INSN_COST);
7715 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7716
7717 ins_encode %{
7718 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7719 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7720 %}
7721
7722 ins_pipe(ialu_reg);
7723 %}
7724
7725 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7726 match(Set dst (ReverseBytesS src));
7727
7728 ins_cost(INSN_COST);
7729 format %{ "rev16w $dst, $src\n\t"
7730 "sbfmw $dst, $dst, #0, #15" %}
7731
7732 ins_encode %{
7733 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7734 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7735 %}
7736
7737 ins_pipe(ialu_reg);
7738 %}
7739
7740 // ============================================================================
7741 // Zero Count Instructions
7742
7743 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7744 match(Set dst (CountLeadingZerosI src));
7745
7746 ins_cost(INSN_COST);
7747 format %{ "clzw $dst, $src" %}
7748 ins_encode %{
7749 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7750 %}
7751
7752 ins_pipe(ialu_reg);
7753 %}
7754
7755 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7756 match(Set dst (CountLeadingZerosL src));
7757
7758 ins_cost(INSN_COST);
7759 format %{ "clz $dst, $src" %}
7760 ins_encode %{
7761 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7762 %}
7763
7764 ins_pipe(ialu_reg);
7765 %}
7766
7767 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7768 match(Set dst (CountTrailingZerosI src));
7769
7770 ins_cost(INSN_COST * 2);
7771 format %{ "rbitw $dst, $src\n\t"
7772 "clzw $dst, $dst" %}
7773 ins_encode %{
7774 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7775 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7776 %}
7777
7778 ins_pipe(ialu_reg);
7779 %}
7780
7781 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7782 match(Set dst (CountTrailingZerosL src));
7783
7784 ins_cost(INSN_COST * 2);
7785 format %{ "rbit $dst, $src\n\t"
7786 "clz $dst, $dst" %}
7787 ins_encode %{
7788 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7789 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7790 %}
7791
7792 ins_pipe(ialu_reg);
7793 %}
7794
7795 //---------- Population Count Instructions -------------------------------------
7796 //
7797
7798 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7799 match(Set dst (PopCountI src));
7800 effect(TEMP tmp);
7801 ins_cost(INSN_COST * 13);
7802
7803 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7804 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7805 "addv $tmp, $tmp\t# vector (8B)\n\t"
7806 "mov $dst, $tmp\t# vector (1D)" %}
7807 ins_encode %{
7808 __ fmovs($tmp$$FloatRegister, $src$$Register);
7809 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7810 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7811 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7812 %}
7813
7814 ins_pipe(pipe_class_default);
7815 %}
7816
7817 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7818 match(Set dst (PopCountI (LoadI mem)));
7819 effect(TEMP tmp);
7820 ins_cost(INSN_COST * 13);
7821
7822 format %{ "ldrs $tmp, $mem\n\t"
7823 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7824 "addv $tmp, $tmp\t# vector (8B)\n\t"
7825 "mov $dst, $tmp\t# vector (1D)" %}
7826 ins_encode %{
7827 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7828 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7829 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7830 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7831 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7832 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7833 %}
7834
7835 ins_pipe(pipe_class_default);
7836 %}
7837
7838 // Note: Long.bitCount(long) returns an int.
7839 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7840 match(Set dst (PopCountL src));
7841 effect(TEMP tmp);
7842 ins_cost(INSN_COST * 13);
7843
7844 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7845 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7846 "addv $tmp, $tmp\t# vector (8B)\n\t"
7847 "mov $dst, $tmp\t# vector (1D)" %}
7848 ins_encode %{
7849 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7850 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7851 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7852 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7853 %}
7854
7855 ins_pipe(pipe_class_default);
7856 %}
7857
7858 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7859 match(Set dst (PopCountL (LoadL mem)));
7860 effect(TEMP tmp);
7861 ins_cost(INSN_COST * 13);
7862
7863 format %{ "ldrd $tmp, $mem\n\t"
7864 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7865 "addv $tmp, $tmp\t# vector (8B)\n\t"
7866 "mov $dst, $tmp\t# vector (1D)" %}
7867 ins_encode %{
7868 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7869 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7870 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7871 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7872 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7873 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7874 %}
7875
7876 ins_pipe(pipe_class_default);
7877 %}
7878
7879 // ============================================================================
7880 // VerifyVectorAlignment Instruction
7881
7882 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7883 match(Set addr (VerifyVectorAlignment addr mask));
7884 effect(KILL cr);
7885 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7886 ins_encode %{
7887 Label Lskip;
7888 // check if masked bits of addr are zero
7889 __ tst($addr$$Register, $mask$$constant);
7890 __ br(Assembler::EQ, Lskip);
7891 __ stop("verify_vector_alignment found a misaligned vector memory access");
7892 __ bind(Lskip);
7893 %}
7894 ins_pipe(pipe_slow);
7895 %}
7896
7897 // ============================================================================
7898 // MemBar Instruction
7899
7900 instruct load_fence() %{
7901 match(LoadFence);
7902 ins_cost(VOLATILE_REF_COST);
7903
7904 format %{ "load_fence" %}
7905
7906 ins_encode %{
7907 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7908 %}
7909 ins_pipe(pipe_serial);
7910 %}
7911
7912 instruct unnecessary_membar_acquire() %{
7913 predicate(unnecessary_acquire(n));
7914 match(MemBarAcquire);
7915 ins_cost(0);
7916
7917 format %{ "membar_acquire (elided)" %}
7918
7919 ins_encode %{
7920 __ block_comment("membar_acquire (elided)");
7921 %}
7922
7923 ins_pipe(pipe_class_empty);
7924 %}
7925
7926 instruct membar_acquire() %{
7927 match(MemBarAcquire);
7928 ins_cost(VOLATILE_REF_COST);
7929
7930 format %{ "membar_acquire\n\t"
7931 "dmb ishld" %}
7932
7933 ins_encode %{
7934 __ block_comment("membar_acquire");
7935 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7936 %}
7937
7938 ins_pipe(pipe_serial);
7939 %}
7940
7941
7942 instruct membar_acquire_lock() %{
7943 match(MemBarAcquireLock);
7944 ins_cost(VOLATILE_REF_COST);
7945
7946 format %{ "membar_acquire_lock (elided)" %}
7947
7948 ins_encode %{
7949 __ block_comment("membar_acquire_lock (elided)");
7950 %}
7951
7952 ins_pipe(pipe_serial);
7953 %}
7954
7955 instruct store_fence() %{
7956 match(StoreFence);
7957 ins_cost(VOLATILE_REF_COST);
7958
7959 format %{ "store_fence" %}
7960
7961 ins_encode %{
7962 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7963 %}
7964 ins_pipe(pipe_serial);
7965 %}
7966
7967 instruct unnecessary_membar_release() %{
7968 predicate(unnecessary_release(n));
7969 match(MemBarRelease);
7970 ins_cost(0);
7971
7972 format %{ "membar_release (elided)" %}
7973
7974 ins_encode %{
7975 __ block_comment("membar_release (elided)");
7976 %}
7977 ins_pipe(pipe_serial);
7978 %}
7979
7980 instruct membar_release() %{
7981 match(MemBarRelease);
7982 ins_cost(VOLATILE_REF_COST);
7983
7984 format %{ "membar_release\n\t"
7985 "dmb ishst\n\tdmb ishld" %}
7986
7987 ins_encode %{
7988 __ block_comment("membar_release");
7989 // These will be merged if AlwaysMergeDMB is enabled.
7990 __ membar(Assembler::StoreStore);
7991 __ membar(Assembler::LoadStore);
7992 %}
7993 ins_pipe(pipe_serial);
7994 %}
7995
7996 instruct membar_storestore() %{
7997 match(MemBarStoreStore);
7998 match(StoreStoreFence);
7999 ins_cost(VOLATILE_REF_COST);
8000
8001 format %{ "MEMBAR-store-store" %}
8002
8003 ins_encode %{
8004 __ membar(Assembler::StoreStore);
8005 %}
8006 ins_pipe(pipe_serial);
8007 %}
8008
8009 instruct membar_release_lock() %{
8010 match(MemBarReleaseLock);
8011 ins_cost(VOLATILE_REF_COST);
8012
8013 format %{ "membar_release_lock (elided)" %}
8014
8015 ins_encode %{
8016 __ block_comment("membar_release_lock (elided)");
8017 %}
8018
8019 ins_pipe(pipe_serial);
8020 %}
8021
8022 instruct membar_storeload() %{
8023 match(MemBarStoreLoad);
8024 ins_cost(VOLATILE_REF_COST*100);
8025
8026 format %{ "MEMBAR-store-load\n\t"
8027 "dmb ish" %}
8028
8029 ins_encode %{
8030 __ block_comment("membar_storeload");
8031 __ membar(Assembler::StoreLoad);
8032 %}
8033
8034 ins_pipe(pipe_serial);
8035 %}
8036
8037 instruct unnecessary_membar_volatile() %{
8038 predicate(unnecessary_volatile(n));
8039 match(MemBarVolatile);
8040 ins_cost(0);
8041
8042 format %{ "membar_volatile (elided)" %}
8043
8044 ins_encode %{
8045 __ block_comment("membar_volatile (elided)");
8046 %}
8047
8048 ins_pipe(pipe_serial);
8049 %}
8050
8051 instruct membar_volatile() %{
8052 match(MemBarVolatile);
8053 ins_cost(VOLATILE_REF_COST*100);
8054
8055 format %{ "membar_volatile\n\t"
8056 "dmb ish"%}
8057
8058 ins_encode %{
8059 __ block_comment("membar_volatile");
8060 __ membar(Assembler::StoreLoad);
8061 %}
8062
8063 ins_pipe(pipe_serial);
8064 %}
8065
8066 instruct membar_full() %{
8067 match(MemBarFull);
8068 ins_cost(VOLATILE_REF_COST*100);
8069
8070 format %{ "membar_full\n\t"
8071 "dmb ish" %}
8072 ins_encode %{
8073 __ block_comment("membar_full");
8074 __ membar(Assembler::AnyAny);
8075 %}
8076
8077 ins_pipe(pipe_serial);
8078 %}
8079
8080 // ============================================================================
8081 // Cast/Convert Instructions
8082
8083 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8084 match(Set dst (CastX2P src));
8085
8086 ins_cost(INSN_COST);
8087 format %{ "mov $dst, $src\t# long -> ptr" %}
8088
8089 ins_encode %{
8090 if ($dst$$reg != $src$$reg) {
8091 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8092 }
8093 %}
8094
8095 ins_pipe(ialu_reg);
8096 %}
8097
8098 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8099 match(Set dst (CastP2X src));
8100
8101 ins_cost(INSN_COST);
8102 format %{ "mov $dst, $src\t# ptr -> long" %}
8103
8104 ins_encode %{
8105 if ($dst$$reg != $src$$reg) {
8106 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8107 }
8108 %}
8109
8110 ins_pipe(ialu_reg);
8111 %}
8112
8113 // Convert oop into int for vectors alignment masking
8114 instruct convP2I(iRegINoSp dst, iRegP src) %{
8115 match(Set dst (ConvL2I (CastP2X src)));
8116
8117 ins_cost(INSN_COST);
8118 format %{ "movw $dst, $src\t# ptr -> int" %}
8119 ins_encode %{
8120 __ movw($dst$$Register, $src$$Register);
8121 %}
8122
8123 ins_pipe(ialu_reg);
8124 %}
8125
8126 // Convert compressed oop into int for vectors alignment masking
8127 // in case of 32bit oops (heap < 4Gb).
8128 instruct convN2I(iRegINoSp dst, iRegN src)
8129 %{
8130 predicate(CompressedOops::shift() == 0);
8131 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8132
8133 ins_cost(INSN_COST);
8134 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8135 ins_encode %{
8136 __ movw($dst$$Register, $src$$Register);
8137 %}
8138
8139 ins_pipe(ialu_reg);
8140 %}
8141
8142
8143 // Convert oop pointer into compressed form
8144 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8145 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8146 match(Set dst (EncodeP src));
8147 effect(KILL cr);
8148 ins_cost(INSN_COST * 3);
8149 format %{ "encode_heap_oop $dst, $src" %}
8150 ins_encode %{
8151 Register s = $src$$Register;
8152 Register d = $dst$$Register;
8153 __ encode_heap_oop(d, s);
8154 %}
8155 ins_pipe(ialu_reg);
8156 %}
8157
8158 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8159 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8160 match(Set dst (EncodeP src));
8161 ins_cost(INSN_COST * 3);
8162 format %{ "encode_heap_oop_not_null $dst, $src" %}
8163 ins_encode %{
8164 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8165 %}
8166 ins_pipe(ialu_reg);
8167 %}
8168
8169 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8170 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8171 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8172 match(Set dst (DecodeN src));
8173 ins_cost(INSN_COST * 3);
8174 format %{ "decode_heap_oop $dst, $src" %}
8175 ins_encode %{
8176 Register s = $src$$Register;
8177 Register d = $dst$$Register;
8178 __ decode_heap_oop(d, s);
8179 %}
8180 ins_pipe(ialu_reg);
8181 %}
8182
8183 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8184 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8185 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8186 match(Set dst (DecodeN src));
8187 ins_cost(INSN_COST * 3);
8188 format %{ "decode_heap_oop_not_null $dst, $src" %}
8189 ins_encode %{
8190 Register s = $src$$Register;
8191 Register d = $dst$$Register;
8192 __ decode_heap_oop_not_null(d, s);
8193 %}
8194 ins_pipe(ialu_reg);
8195 %}
8196
8197 // n.b. AArch64 implementations of encode_klass_not_null and
8198 // decode_klass_not_null do not modify the flags register so, unlike
8199 // Intel, we don't kill CR as a side effect here
8200
8201 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8202 match(Set dst (EncodePKlass src));
8203
8204 ins_cost(INSN_COST * 3);
8205 format %{ "encode_klass_not_null $dst,$src" %}
8206
8207 ins_encode %{
8208 Register src_reg = as_Register($src$$reg);
8209 Register dst_reg = as_Register($dst$$reg);
8210 __ encode_klass_not_null(dst_reg, src_reg);
8211 %}
8212
8213 ins_pipe(ialu_reg);
8214 %}
8215
8216 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8217 match(Set dst (DecodeNKlass src));
8218
8219 ins_cost(INSN_COST * 3);
8220 format %{ "decode_klass_not_null $dst,$src" %}
8221
8222 ins_encode %{
8223 Register src_reg = as_Register($src$$reg);
8224 Register dst_reg = as_Register($dst$$reg);
8225 if (dst_reg != src_reg) {
8226 __ decode_klass_not_null(dst_reg, src_reg);
8227 } else {
8228 __ decode_klass_not_null(dst_reg);
8229 }
8230 %}
8231
8232 ins_pipe(ialu_reg);
8233 %}
8234
8235 instruct checkCastPP(iRegPNoSp dst)
8236 %{
8237 match(Set dst (CheckCastPP dst));
8238
8239 size(0);
8240 format %{ "# checkcastPP of $dst" %}
8241 ins_encode(/* empty encoding */);
8242 ins_pipe(pipe_class_empty);
8243 %}
8244
8245 instruct castPP(iRegPNoSp dst)
8246 %{
8247 match(Set dst (CastPP dst));
8248
8249 size(0);
8250 format %{ "# castPP of $dst" %}
8251 ins_encode(/* empty encoding */);
8252 ins_pipe(pipe_class_empty);
8253 %}
8254
8255 instruct castII(iRegI dst)
8256 %{
8257 predicate(VerifyConstraintCasts == 0);
8258 match(Set dst (CastII dst));
8259
8260 size(0);
8261 format %{ "# castII of $dst" %}
8262 ins_encode(/* empty encoding */);
8263 ins_cost(0);
8264 ins_pipe(pipe_class_empty);
8265 %}
8266
8267 instruct castII_checked(iRegI dst, rFlagsReg cr)
8268 %{
8269 predicate(VerifyConstraintCasts > 0);
8270 match(Set dst (CastII dst));
8271 effect(KILL cr);
8272
8273 format %{ "# castII_checked of $dst" %}
8274 ins_encode %{
8275 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8276 %}
8277 ins_pipe(pipe_slow);
8278 %}
8279
8280 instruct castLL(iRegL dst)
8281 %{
8282 predicate(VerifyConstraintCasts == 0);
8283 match(Set dst (CastLL dst));
8284
8285 size(0);
8286 format %{ "# castLL of $dst" %}
8287 ins_encode(/* empty encoding */);
8288 ins_cost(0);
8289 ins_pipe(pipe_class_empty);
8290 %}
8291
8292 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8293 %{
8294 predicate(VerifyConstraintCasts > 0);
8295 match(Set dst (CastLL dst));
8296 effect(KILL cr);
8297
8298 format %{ "# castLL_checked of $dst" %}
8299 ins_encode %{
8300 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8301 %}
8302 ins_pipe(pipe_slow);
8303 %}
8304
8305 instruct castHH(vRegF dst)
8306 %{
8307 match(Set dst (CastHH dst));
8308 size(0);
8309 format %{ "# castHH of $dst" %}
8310 ins_encode(/* empty encoding */);
8311 ins_cost(0);
8312 ins_pipe(pipe_class_empty);
8313 %}
8314
8315 instruct castFF(vRegF dst)
8316 %{
8317 match(Set dst (CastFF dst));
8318
8319 size(0);
8320 format %{ "# castFF of $dst" %}
8321 ins_encode(/* empty encoding */);
8322 ins_cost(0);
8323 ins_pipe(pipe_class_empty);
8324 %}
8325
8326 instruct castDD(vRegD dst)
8327 %{
8328 match(Set dst (CastDD dst));
8329
8330 size(0);
8331 format %{ "# castDD of $dst" %}
8332 ins_encode(/* empty encoding */);
8333 ins_cost(0);
8334 ins_pipe(pipe_class_empty);
8335 %}
8336
8337 instruct castVV(vReg dst)
8338 %{
8339 match(Set dst (CastVV dst));
8340
8341 size(0);
8342 format %{ "# castVV of $dst" %}
8343 ins_encode(/* empty encoding */);
8344 ins_cost(0);
8345 ins_pipe(pipe_class_empty);
8346 %}
8347
8348 instruct castVVMask(pRegGov dst)
8349 %{
8350 match(Set dst (CastVV dst));
8351
8352 size(0);
8353 format %{ "# castVV of $dst" %}
8354 ins_encode(/* empty encoding */);
8355 ins_cost(0);
8356 ins_pipe(pipe_class_empty);
8357 %}
8358
8359 // Manifest a CmpU result in an integer register.
8360 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8361 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8362 %{
8363 match(Set dst (CmpU3 src1 src2));
8364 effect(KILL flags);
8365
8366 ins_cost(INSN_COST * 3);
8367 format %{
8368 "cmpw $src1, $src2\n\t"
8369 "csetw $dst, ne\n\t"
8370 "cnegw $dst, lo\t# CmpU3(reg)"
8371 %}
8372 ins_encode %{
8373 __ cmpw($src1$$Register, $src2$$Register);
8374 __ csetw($dst$$Register, Assembler::NE);
8375 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8376 %}
8377
8378 ins_pipe(pipe_class_default);
8379 %}
8380
8381 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8382 %{
8383 match(Set dst (CmpU3 src1 src2));
8384 effect(KILL flags);
8385
8386 ins_cost(INSN_COST * 3);
8387 format %{
8388 "subsw zr, $src1, $src2\n\t"
8389 "csetw $dst, ne\n\t"
8390 "cnegw $dst, lo\t# CmpU3(imm)"
8391 %}
8392 ins_encode %{
8393 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8394 __ csetw($dst$$Register, Assembler::NE);
8395 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8396 %}
8397
8398 ins_pipe(pipe_class_default);
8399 %}
8400
8401 // Manifest a CmpUL result in an integer register.
8402 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8403 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8404 %{
8405 match(Set dst (CmpUL3 src1 src2));
8406 effect(KILL flags);
8407
8408 ins_cost(INSN_COST * 3);
8409 format %{
8410 "cmp $src1, $src2\n\t"
8411 "csetw $dst, ne\n\t"
8412 "cnegw $dst, lo\t# CmpUL3(reg)"
8413 %}
8414 ins_encode %{
8415 __ cmp($src1$$Register, $src2$$Register);
8416 __ csetw($dst$$Register, Assembler::NE);
8417 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8418 %}
8419
8420 ins_pipe(pipe_class_default);
8421 %}
8422
8423 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8424 %{
8425 match(Set dst (CmpUL3 src1 src2));
8426 effect(KILL flags);
8427
8428 ins_cost(INSN_COST * 3);
8429 format %{
8430 "subs zr, $src1, $src2\n\t"
8431 "csetw $dst, ne\n\t"
8432 "cnegw $dst, lo\t# CmpUL3(imm)"
8433 %}
8434 ins_encode %{
8435 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8436 __ csetw($dst$$Register, Assembler::NE);
8437 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8438 %}
8439
8440 ins_pipe(pipe_class_default);
8441 %}
8442
8443 // Manifest a CmpL result in an integer register.
8444 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8445 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8446 %{
8447 match(Set dst (CmpL3 src1 src2));
8448 effect(KILL flags);
8449
8450 ins_cost(INSN_COST * 3);
8451 format %{
8452 "cmp $src1, $src2\n\t"
8453 "csetw $dst, ne\n\t"
8454 "cnegw $dst, lt\t# CmpL3(reg)"
8455 %}
8456 ins_encode %{
8457 __ cmp($src1$$Register, $src2$$Register);
8458 __ csetw($dst$$Register, Assembler::NE);
8459 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8460 %}
8461
8462 ins_pipe(pipe_class_default);
8463 %}
8464
8465 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8466 %{
8467 match(Set dst (CmpL3 src1 src2));
8468 effect(KILL flags);
8469
8470 ins_cost(INSN_COST * 3);
8471 format %{
8472 "subs zr, $src1, $src2\n\t"
8473 "csetw $dst, ne\n\t"
8474 "cnegw $dst, lt\t# CmpL3(imm)"
8475 %}
8476 ins_encode %{
8477 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8478 __ csetw($dst$$Register, Assembler::NE);
8479 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8480 %}
8481
8482 ins_pipe(pipe_class_default);
8483 %}
8484
8485 // ============================================================================
8486 // Conditional Move Instructions
8487
8488 // n.b. we have identical rules for both a signed compare op (cmpOp)
8489 // and an unsigned compare op (cmpOpU). it would be nice if we could
8490 // define an op class which merged both inputs and use it to type the
8491 // argument to a single rule. unfortunatelyt his fails because the
8492 // opclass does not live up to the COND_INTER interface of its
8493 // component operands. When the generic code tries to negate the
8494 // operand it ends up running the generci Machoper::negate method
8495 // which throws a ShouldNotHappen. So, we have to provide two flavours
8496 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8497
8498 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8499 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8500
8501 ins_cost(INSN_COST * 2);
8502 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8503
8504 ins_encode %{
8505 __ cselw(as_Register($dst$$reg),
8506 as_Register($src2$$reg),
8507 as_Register($src1$$reg),
8508 (Assembler::Condition)$cmp$$cmpcode);
8509 %}
8510
8511 ins_pipe(icond_reg_reg);
8512 %}
8513
8514 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8515 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8516
8517 ins_cost(INSN_COST * 2);
8518 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8519
8520 ins_encode %{
8521 __ cselw(as_Register($dst$$reg),
8522 as_Register($src2$$reg),
8523 as_Register($src1$$reg),
8524 (Assembler::Condition)$cmp$$cmpcode);
8525 %}
8526
8527 ins_pipe(icond_reg_reg);
8528 %}
8529
8530 // special cases where one arg is zero
8531
8532 // n.b. this is selected in preference to the rule above because it
8533 // avoids loading constant 0 into a source register
8534
8535 // TODO
8536 // we ought only to be able to cull one of these variants as the ideal
8537 // transforms ought always to order the zero consistently (to left/right?)
8538
8539 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8540 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8541
8542 ins_cost(INSN_COST * 2);
8543 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8544
8545 ins_encode %{
8546 __ cselw(as_Register($dst$$reg),
8547 as_Register($src$$reg),
8548 zr,
8549 (Assembler::Condition)$cmp$$cmpcode);
8550 %}
8551
8552 ins_pipe(icond_reg);
8553 %}
8554
8555 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8556 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8557
8558 ins_cost(INSN_COST * 2);
8559 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8560
8561 ins_encode %{
8562 __ cselw(as_Register($dst$$reg),
8563 as_Register($src$$reg),
8564 zr,
8565 (Assembler::Condition)$cmp$$cmpcode);
8566 %}
8567
8568 ins_pipe(icond_reg);
8569 %}
8570
8571 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8572 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8573
8574 ins_cost(INSN_COST * 2);
8575 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8576
8577 ins_encode %{
8578 __ cselw(as_Register($dst$$reg),
8579 zr,
8580 as_Register($src$$reg),
8581 (Assembler::Condition)$cmp$$cmpcode);
8582 %}
8583
8584 ins_pipe(icond_reg);
8585 %}
8586
8587 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8588 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8589
8590 ins_cost(INSN_COST * 2);
8591 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8592
8593 ins_encode %{
8594 __ cselw(as_Register($dst$$reg),
8595 zr,
8596 as_Register($src$$reg),
8597 (Assembler::Condition)$cmp$$cmpcode);
8598 %}
8599
8600 ins_pipe(icond_reg);
8601 %}
8602
8603 // special case for creating a boolean 0 or 1
8604
8605 // n.b. this is selected in preference to the rule above because it
8606 // avoids loading constants 0 and 1 into a source register
8607
8608 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8609 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8610
8611 ins_cost(INSN_COST * 2);
8612 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8613
8614 ins_encode %{
8615 // equivalently
8616 // cset(as_Register($dst$$reg),
8617 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8618 __ csincw(as_Register($dst$$reg),
8619 zr,
8620 zr,
8621 (Assembler::Condition)$cmp$$cmpcode);
8622 %}
8623
8624 ins_pipe(icond_none);
8625 %}
8626
8627 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8628 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8629
8630 ins_cost(INSN_COST * 2);
8631 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8632
8633 ins_encode %{
8634 // equivalently
8635 // cset(as_Register($dst$$reg),
8636 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8637 __ csincw(as_Register($dst$$reg),
8638 zr,
8639 zr,
8640 (Assembler::Condition)$cmp$$cmpcode);
8641 %}
8642
8643 ins_pipe(icond_none);
8644 %}
8645
8646 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8647 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8648
8649 ins_cost(INSN_COST * 2);
8650 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8651
8652 ins_encode %{
8653 __ csel(as_Register($dst$$reg),
8654 as_Register($src2$$reg),
8655 as_Register($src1$$reg),
8656 (Assembler::Condition)$cmp$$cmpcode);
8657 %}
8658
8659 ins_pipe(icond_reg_reg);
8660 %}
8661
8662 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8663 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8664
8665 ins_cost(INSN_COST * 2);
8666 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8667
8668 ins_encode %{
8669 __ csel(as_Register($dst$$reg),
8670 as_Register($src2$$reg),
8671 as_Register($src1$$reg),
8672 (Assembler::Condition)$cmp$$cmpcode);
8673 %}
8674
8675 ins_pipe(icond_reg_reg);
8676 %}
8677
8678 // special cases where one arg is zero
8679
8680 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8681 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8682
8683 ins_cost(INSN_COST * 2);
8684 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8685
8686 ins_encode %{
8687 __ csel(as_Register($dst$$reg),
8688 zr,
8689 as_Register($src$$reg),
8690 (Assembler::Condition)$cmp$$cmpcode);
8691 %}
8692
8693 ins_pipe(icond_reg);
8694 %}
8695
8696 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8697 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8698
8699 ins_cost(INSN_COST * 2);
8700 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8701
8702 ins_encode %{
8703 __ csel(as_Register($dst$$reg),
8704 zr,
8705 as_Register($src$$reg),
8706 (Assembler::Condition)$cmp$$cmpcode);
8707 %}
8708
8709 ins_pipe(icond_reg);
8710 %}
8711
8712 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8713 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8714
8715 ins_cost(INSN_COST * 2);
8716 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8717
8718 ins_encode %{
8719 __ csel(as_Register($dst$$reg),
8720 as_Register($src$$reg),
8721 zr,
8722 (Assembler::Condition)$cmp$$cmpcode);
8723 %}
8724
8725 ins_pipe(icond_reg);
8726 %}
8727
8728 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8729 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8730
8731 ins_cost(INSN_COST * 2);
8732 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8733
8734 ins_encode %{
8735 __ csel(as_Register($dst$$reg),
8736 as_Register($src$$reg),
8737 zr,
8738 (Assembler::Condition)$cmp$$cmpcode);
8739 %}
8740
8741 ins_pipe(icond_reg);
8742 %}
8743
8744 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8745 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8746
8747 ins_cost(INSN_COST * 2);
8748 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8749
8750 ins_encode %{
8751 __ csel(as_Register($dst$$reg),
8752 as_Register($src2$$reg),
8753 as_Register($src1$$reg),
8754 (Assembler::Condition)$cmp$$cmpcode);
8755 %}
8756
8757 ins_pipe(icond_reg_reg);
8758 %}
8759
8760 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8761 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8762
8763 ins_cost(INSN_COST * 2);
8764 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8765
8766 ins_encode %{
8767 __ csel(as_Register($dst$$reg),
8768 as_Register($src2$$reg),
8769 as_Register($src1$$reg),
8770 (Assembler::Condition)$cmp$$cmpcode);
8771 %}
8772
8773 ins_pipe(icond_reg_reg);
8774 %}
8775
8776 // special cases where one arg is zero
8777
8778 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8779 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8780
8781 ins_cost(INSN_COST * 2);
8782 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8783
8784 ins_encode %{
8785 __ csel(as_Register($dst$$reg),
8786 zr,
8787 as_Register($src$$reg),
8788 (Assembler::Condition)$cmp$$cmpcode);
8789 %}
8790
8791 ins_pipe(icond_reg);
8792 %}
8793
8794 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8795 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8796
8797 ins_cost(INSN_COST * 2);
8798 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8799
8800 ins_encode %{
8801 __ csel(as_Register($dst$$reg),
8802 zr,
8803 as_Register($src$$reg),
8804 (Assembler::Condition)$cmp$$cmpcode);
8805 %}
8806
8807 ins_pipe(icond_reg);
8808 %}
8809
8810 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8811 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8812
8813 ins_cost(INSN_COST * 2);
8814 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8815
8816 ins_encode %{
8817 __ csel(as_Register($dst$$reg),
8818 as_Register($src$$reg),
8819 zr,
8820 (Assembler::Condition)$cmp$$cmpcode);
8821 %}
8822
8823 ins_pipe(icond_reg);
8824 %}
8825
8826 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8827 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8828
8829 ins_cost(INSN_COST * 2);
8830 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8831
8832 ins_encode %{
8833 __ csel(as_Register($dst$$reg),
8834 as_Register($src$$reg),
8835 zr,
8836 (Assembler::Condition)$cmp$$cmpcode);
8837 %}
8838
8839 ins_pipe(icond_reg);
8840 %}
8841
8842 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8843 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8844
8845 ins_cost(INSN_COST * 2);
8846 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8847
8848 ins_encode %{
8849 __ cselw(as_Register($dst$$reg),
8850 as_Register($src2$$reg),
8851 as_Register($src1$$reg),
8852 (Assembler::Condition)$cmp$$cmpcode);
8853 %}
8854
8855 ins_pipe(icond_reg_reg);
8856 %}
8857
8858 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8859 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8860
8861 ins_cost(INSN_COST * 2);
8862 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8863
8864 ins_encode %{
8865 __ cselw(as_Register($dst$$reg),
8866 as_Register($src2$$reg),
8867 as_Register($src1$$reg),
8868 (Assembler::Condition)$cmp$$cmpcode);
8869 %}
8870
8871 ins_pipe(icond_reg_reg);
8872 %}
8873
8874 // special cases where one arg is zero
8875
8876 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8877 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8878
8879 ins_cost(INSN_COST * 2);
8880 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8881
8882 ins_encode %{
8883 __ cselw(as_Register($dst$$reg),
8884 zr,
8885 as_Register($src$$reg),
8886 (Assembler::Condition)$cmp$$cmpcode);
8887 %}
8888
8889 ins_pipe(icond_reg);
8890 %}
8891
8892 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8893 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8894
8895 ins_cost(INSN_COST * 2);
8896 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8897
8898 ins_encode %{
8899 __ cselw(as_Register($dst$$reg),
8900 zr,
8901 as_Register($src$$reg),
8902 (Assembler::Condition)$cmp$$cmpcode);
8903 %}
8904
8905 ins_pipe(icond_reg);
8906 %}
8907
8908 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8909 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8910
8911 ins_cost(INSN_COST * 2);
8912 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8913
8914 ins_encode %{
8915 __ cselw(as_Register($dst$$reg),
8916 as_Register($src$$reg),
8917 zr,
8918 (Assembler::Condition)$cmp$$cmpcode);
8919 %}
8920
8921 ins_pipe(icond_reg);
8922 %}
8923
8924 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8925 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8926
8927 ins_cost(INSN_COST * 2);
8928 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8929
8930 ins_encode %{
8931 __ cselw(as_Register($dst$$reg),
8932 as_Register($src$$reg),
8933 zr,
8934 (Assembler::Condition)$cmp$$cmpcode);
8935 %}
8936
8937 ins_pipe(icond_reg);
8938 %}
8939
8940 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8941 %{
8942 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8943
8944 ins_cost(INSN_COST * 3);
8945
8946 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8947 ins_encode %{
8948 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8949 __ fcsels(as_FloatRegister($dst$$reg),
8950 as_FloatRegister($src2$$reg),
8951 as_FloatRegister($src1$$reg),
8952 cond);
8953 %}
8954
8955 ins_pipe(fp_cond_reg_reg_s);
8956 %}
8957
8958 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8959 %{
8960 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8961
8962 ins_cost(INSN_COST * 3);
8963
8964 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8965 ins_encode %{
8966 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8967 __ fcsels(as_FloatRegister($dst$$reg),
8968 as_FloatRegister($src2$$reg),
8969 as_FloatRegister($src1$$reg),
8970 cond);
8971 %}
8972
8973 ins_pipe(fp_cond_reg_reg_s);
8974 %}
8975
8976 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8977 %{
8978 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8979
8980 ins_cost(INSN_COST * 3);
8981
8982 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8983 ins_encode %{
8984 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8985 __ fcseld(as_FloatRegister($dst$$reg),
8986 as_FloatRegister($src2$$reg),
8987 as_FloatRegister($src1$$reg),
8988 cond);
8989 %}
8990
8991 ins_pipe(fp_cond_reg_reg_d);
8992 %}
8993
8994 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8995 %{
8996 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8997
8998 ins_cost(INSN_COST * 3);
8999
9000 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9001 ins_encode %{
9002 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9003 __ fcseld(as_FloatRegister($dst$$reg),
9004 as_FloatRegister($src2$$reg),
9005 as_FloatRegister($src1$$reg),
9006 cond);
9007 %}
9008
9009 ins_pipe(fp_cond_reg_reg_d);
9010 %}
9011
9012 // ============================================================================
9013 // Arithmetic Instructions
9014 //
9015
9016 // Integer Addition
9017
9018 // TODO
9019 // these currently employ operations which do not set CR and hence are
9020 // not flagged as killing CR but we would like to isolate the cases
9021 // where we want to set flags from those where we don't. need to work
9022 // out how to do that.
9023
9024 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9025 match(Set dst (AddI src1 src2));
9026
9027 ins_cost(INSN_COST);
9028 format %{ "addw $dst, $src1, $src2" %}
9029
9030 ins_encode %{
9031 __ addw(as_Register($dst$$reg),
9032 as_Register($src1$$reg),
9033 as_Register($src2$$reg));
9034 %}
9035
9036 ins_pipe(ialu_reg_reg);
9037 %}
9038
9039 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9040 match(Set dst (AddI src1 src2));
9041
9042 ins_cost(INSN_COST);
9043 format %{ "addw $dst, $src1, $src2" %}
9044
9045 // use opcode to indicate that this is an add not a sub
9046 opcode(0x0);
9047
9048 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9049
9050 ins_pipe(ialu_reg_imm);
9051 %}
9052
9053 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9054 match(Set dst (AddI (ConvL2I src1) src2));
9055
9056 ins_cost(INSN_COST);
9057 format %{ "addw $dst, $src1, $src2" %}
9058
9059 // use opcode to indicate that this is an add not a sub
9060 opcode(0x0);
9061
9062 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9063
9064 ins_pipe(ialu_reg_imm);
9065 %}
9066
9067 // Pointer Addition
9068 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9069 match(Set dst (AddP src1 src2));
9070
9071 ins_cost(INSN_COST);
9072 format %{ "add $dst, $src1, $src2\t# ptr" %}
9073
9074 ins_encode %{
9075 __ add(as_Register($dst$$reg),
9076 as_Register($src1$$reg),
9077 as_Register($src2$$reg));
9078 %}
9079
9080 ins_pipe(ialu_reg_reg);
9081 %}
9082
9083 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9084 match(Set dst (AddP src1 (ConvI2L src2)));
9085
9086 ins_cost(1.9 * INSN_COST);
9087 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9088
9089 ins_encode %{
9090 __ add(as_Register($dst$$reg),
9091 as_Register($src1$$reg),
9092 as_Register($src2$$reg), ext::sxtw);
9093 %}
9094
9095 ins_pipe(ialu_reg_reg);
9096 %}
9097
9098 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9099 match(Set dst (AddP src1 (LShiftL src2 scale)));
9100
9101 ins_cost(1.9 * INSN_COST);
9102 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9103
9104 ins_encode %{
9105 __ lea(as_Register($dst$$reg),
9106 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9107 Address::lsl($scale$$constant)));
9108 %}
9109
9110 ins_pipe(ialu_reg_reg_shift);
9111 %}
9112
9113 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9114 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9115
9116 ins_cost(1.9 * INSN_COST);
9117 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9118
9119 ins_encode %{
9120 __ lea(as_Register($dst$$reg),
9121 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9122 Address::sxtw($scale$$constant)));
9123 %}
9124
9125 ins_pipe(ialu_reg_reg_shift);
9126 %}
9127
9128 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9129 match(Set dst (LShiftL (ConvI2L src) scale));
9130
9131 ins_cost(INSN_COST);
9132 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9133
9134 ins_encode %{
9135 __ sbfiz(as_Register($dst$$reg),
9136 as_Register($src$$reg),
9137 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9138 %}
9139
9140 ins_pipe(ialu_reg_shift);
9141 %}
9142
9143 // Pointer Immediate Addition
9144 // n.b. this needs to be more expensive than using an indirect memory
9145 // operand
9146 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9147 match(Set dst (AddP src1 src2));
9148
9149 ins_cost(INSN_COST);
9150 format %{ "add $dst, $src1, $src2\t# ptr" %}
9151
9152 // use opcode to indicate that this is an add not a sub
9153 opcode(0x0);
9154
9155 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9156
9157 ins_pipe(ialu_reg_imm);
9158 %}
9159
9160 // Long Addition
9161 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9162
9163 match(Set dst (AddL src1 src2));
9164
9165 ins_cost(INSN_COST);
9166 format %{ "add $dst, $src1, $src2" %}
9167
9168 ins_encode %{
9169 __ add(as_Register($dst$$reg),
9170 as_Register($src1$$reg),
9171 as_Register($src2$$reg));
9172 %}
9173
9174 ins_pipe(ialu_reg_reg);
9175 %}
9176
9177 // No constant pool entries requiredLong Immediate Addition.
9178 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9179 match(Set dst (AddL src1 src2));
9180
9181 ins_cost(INSN_COST);
9182 format %{ "add $dst, $src1, $src2" %}
9183
9184 // use opcode to indicate that this is an add not a sub
9185 opcode(0x0);
9186
9187 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9188
9189 ins_pipe(ialu_reg_imm);
9190 %}
9191
9192 // Integer Subtraction
9193 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9194 match(Set dst (SubI src1 src2));
9195
9196 ins_cost(INSN_COST);
9197 format %{ "subw $dst, $src1, $src2" %}
9198
9199 ins_encode %{
9200 __ subw(as_Register($dst$$reg),
9201 as_Register($src1$$reg),
9202 as_Register($src2$$reg));
9203 %}
9204
9205 ins_pipe(ialu_reg_reg);
9206 %}
9207
9208 // Immediate Subtraction
9209 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9210 match(Set dst (SubI src1 src2));
9211
9212 ins_cost(INSN_COST);
9213 format %{ "subw $dst, $src1, $src2" %}
9214
9215 // use opcode to indicate that this is a sub not an add
9216 opcode(0x1);
9217
9218 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9219
9220 ins_pipe(ialu_reg_imm);
9221 %}
9222
9223 // Long Subtraction
9224 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9225
9226 match(Set dst (SubL src1 src2));
9227
9228 ins_cost(INSN_COST);
9229 format %{ "sub $dst, $src1, $src2" %}
9230
9231 ins_encode %{
9232 __ sub(as_Register($dst$$reg),
9233 as_Register($src1$$reg),
9234 as_Register($src2$$reg));
9235 %}
9236
9237 ins_pipe(ialu_reg_reg);
9238 %}
9239
9240 // No constant pool entries requiredLong Immediate Subtraction.
9241 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9242 match(Set dst (SubL src1 src2));
9243
9244 ins_cost(INSN_COST);
9245 format %{ "sub$dst, $src1, $src2" %}
9246
9247 // use opcode to indicate that this is a sub not an add
9248 opcode(0x1);
9249
9250 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9251
9252 ins_pipe(ialu_reg_imm);
9253 %}
9254
9255 // Integer Negation (special case for sub)
9256
9257 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9258 match(Set dst (SubI zero src));
9259
9260 ins_cost(INSN_COST);
9261 format %{ "negw $dst, $src\t# int" %}
9262
9263 ins_encode %{
9264 __ negw(as_Register($dst$$reg),
9265 as_Register($src$$reg));
9266 %}
9267
9268 ins_pipe(ialu_reg);
9269 %}
9270
9271 // Long Negation
9272
9273 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9274 match(Set dst (SubL zero src));
9275
9276 ins_cost(INSN_COST);
9277 format %{ "neg $dst, $src\t# long" %}
9278
9279 ins_encode %{
9280 __ neg(as_Register($dst$$reg),
9281 as_Register($src$$reg));
9282 %}
9283
9284 ins_pipe(ialu_reg);
9285 %}
9286
9287 // Integer Multiply
9288
9289 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9290 match(Set dst (MulI src1 src2));
9291
9292 ins_cost(INSN_COST * 3);
9293 format %{ "mulw $dst, $src1, $src2" %}
9294
9295 ins_encode %{
9296 __ mulw(as_Register($dst$$reg),
9297 as_Register($src1$$reg),
9298 as_Register($src2$$reg));
9299 %}
9300
9301 ins_pipe(imul_reg_reg);
9302 %}
9303
9304 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9305 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9306
9307 ins_cost(INSN_COST * 3);
9308 format %{ "smull $dst, $src1, $src2" %}
9309
9310 ins_encode %{
9311 __ smull(as_Register($dst$$reg),
9312 as_Register($src1$$reg),
9313 as_Register($src2$$reg));
9314 %}
9315
9316 ins_pipe(imul_reg_reg);
9317 %}
9318
9319 // Long Multiply
9320
9321 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9322 match(Set dst (MulL src1 src2));
9323
9324 ins_cost(INSN_COST * 5);
9325 format %{ "mul $dst, $src1, $src2" %}
9326
9327 ins_encode %{
9328 __ mul(as_Register($dst$$reg),
9329 as_Register($src1$$reg),
9330 as_Register($src2$$reg));
9331 %}
9332
9333 ins_pipe(lmul_reg_reg);
9334 %}
9335
9336 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9337 %{
9338 match(Set dst (MulHiL src1 src2));
9339
9340 ins_cost(INSN_COST * 7);
9341 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9342
9343 ins_encode %{
9344 __ smulh(as_Register($dst$$reg),
9345 as_Register($src1$$reg),
9346 as_Register($src2$$reg));
9347 %}
9348
9349 ins_pipe(lmul_reg_reg);
9350 %}
9351
9352 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9353 %{
9354 match(Set dst (UMulHiL src1 src2));
9355
9356 ins_cost(INSN_COST * 7);
9357 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9358
9359 ins_encode %{
9360 __ umulh(as_Register($dst$$reg),
9361 as_Register($src1$$reg),
9362 as_Register($src2$$reg));
9363 %}
9364
9365 ins_pipe(lmul_reg_reg);
9366 %}
9367
9368 // Combined Integer Multiply & Add/Sub
9369
9370 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9371 match(Set dst (AddI src3 (MulI src1 src2)));
9372
9373 ins_cost(INSN_COST * 3);
9374 format %{ "madd $dst, $src1, $src2, $src3" %}
9375
9376 ins_encode %{
9377 __ maddw(as_Register($dst$$reg),
9378 as_Register($src1$$reg),
9379 as_Register($src2$$reg),
9380 as_Register($src3$$reg));
9381 %}
9382
9383 ins_pipe(imac_reg_reg);
9384 %}
9385
9386 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9387 match(Set dst (SubI src3 (MulI src1 src2)));
9388
9389 ins_cost(INSN_COST * 3);
9390 format %{ "msub $dst, $src1, $src2, $src3" %}
9391
9392 ins_encode %{
9393 __ msubw(as_Register($dst$$reg),
9394 as_Register($src1$$reg),
9395 as_Register($src2$$reg),
9396 as_Register($src3$$reg));
9397 %}
9398
9399 ins_pipe(imac_reg_reg);
9400 %}
9401
9402 // Combined Integer Multiply & Neg
9403
9404 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9405 match(Set dst (MulI (SubI zero src1) src2));
9406
9407 ins_cost(INSN_COST * 3);
9408 format %{ "mneg $dst, $src1, $src2" %}
9409
9410 ins_encode %{
9411 __ mnegw(as_Register($dst$$reg),
9412 as_Register($src1$$reg),
9413 as_Register($src2$$reg));
9414 %}
9415
9416 ins_pipe(imac_reg_reg);
9417 %}
9418
9419 // Combined Long Multiply & Add/Sub
9420
9421 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9422 match(Set dst (AddL src3 (MulL src1 src2)));
9423
9424 ins_cost(INSN_COST * 5);
9425 format %{ "madd $dst, $src1, $src2, $src3" %}
9426
9427 ins_encode %{
9428 __ madd(as_Register($dst$$reg),
9429 as_Register($src1$$reg),
9430 as_Register($src2$$reg),
9431 as_Register($src3$$reg));
9432 %}
9433
9434 ins_pipe(lmac_reg_reg);
9435 %}
9436
9437 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9438 match(Set dst (SubL src3 (MulL src1 src2)));
9439
9440 ins_cost(INSN_COST * 5);
9441 format %{ "msub $dst, $src1, $src2, $src3" %}
9442
9443 ins_encode %{
9444 __ msub(as_Register($dst$$reg),
9445 as_Register($src1$$reg),
9446 as_Register($src2$$reg),
9447 as_Register($src3$$reg));
9448 %}
9449
9450 ins_pipe(lmac_reg_reg);
9451 %}
9452
9453 // Combined Long Multiply & Neg
9454
9455 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9456 match(Set dst (MulL (SubL zero src1) src2));
9457
9458 ins_cost(INSN_COST * 5);
9459 format %{ "mneg $dst, $src1, $src2" %}
9460
9461 ins_encode %{
9462 __ mneg(as_Register($dst$$reg),
9463 as_Register($src1$$reg),
9464 as_Register($src2$$reg));
9465 %}
9466
9467 ins_pipe(lmac_reg_reg);
9468 %}
9469
9470 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9471
9472 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9473 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9474
9475 ins_cost(INSN_COST * 3);
9476 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9477
9478 ins_encode %{
9479 __ smaddl(as_Register($dst$$reg),
9480 as_Register($src1$$reg),
9481 as_Register($src2$$reg),
9482 as_Register($src3$$reg));
9483 %}
9484
9485 ins_pipe(imac_reg_reg);
9486 %}
9487
9488 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9489 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9490
9491 ins_cost(INSN_COST * 3);
9492 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9493
9494 ins_encode %{
9495 __ smsubl(as_Register($dst$$reg),
9496 as_Register($src1$$reg),
9497 as_Register($src2$$reg),
9498 as_Register($src3$$reg));
9499 %}
9500
9501 ins_pipe(imac_reg_reg);
9502 %}
9503
9504 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9505 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9506
9507 ins_cost(INSN_COST * 3);
9508 format %{ "smnegl $dst, $src1, $src2" %}
9509
9510 ins_encode %{
9511 __ smnegl(as_Register($dst$$reg),
9512 as_Register($src1$$reg),
9513 as_Register($src2$$reg));
9514 %}
9515
9516 ins_pipe(imac_reg_reg);
9517 %}
9518
9519 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9520
9521 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9522 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9523
9524 ins_cost(INSN_COST * 5);
9525 format %{ "mulw rscratch1, $src1, $src2\n\t"
9526 "maddw $dst, $src3, $src4, rscratch1" %}
9527
9528 ins_encode %{
9529 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9530 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9531
9532 ins_pipe(imac_reg_reg);
9533 %}
9534
9535 // Integer Divide
9536
9537 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9538 match(Set dst (DivI src1 src2));
9539
9540 ins_cost(INSN_COST * 19);
9541 format %{ "sdivw $dst, $src1, $src2" %}
9542
9543 ins_encode(aarch64_enc_divw(dst, src1, src2));
9544 ins_pipe(idiv_reg_reg);
9545 %}
9546
9547 // Long Divide
9548
9549 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9550 match(Set dst (DivL src1 src2));
9551
9552 ins_cost(INSN_COST * 35);
9553 format %{ "sdiv $dst, $src1, $src2" %}
9554
9555 ins_encode(aarch64_enc_div(dst, src1, src2));
9556 ins_pipe(ldiv_reg_reg);
9557 %}
9558
9559 // Integer Remainder
9560
9561 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9562 match(Set dst (ModI src1 src2));
9563
9564 ins_cost(INSN_COST * 22);
9565 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9566 "msubw $dst, rscratch1, $src2, $src1" %}
9567
9568 ins_encode(aarch64_enc_modw(dst, src1, src2));
9569 ins_pipe(idiv_reg_reg);
9570 %}
9571
9572 // Long Remainder
9573
9574 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9575 match(Set dst (ModL src1 src2));
9576
9577 ins_cost(INSN_COST * 38);
9578 format %{ "sdiv rscratch1, $src1, $src2\n"
9579 "msub $dst, rscratch1, $src2, $src1" %}
9580
9581 ins_encode(aarch64_enc_mod(dst, src1, src2));
9582 ins_pipe(ldiv_reg_reg);
9583 %}
9584
9585 // Unsigned Integer Divide
9586
9587 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9588 match(Set dst (UDivI src1 src2));
9589
9590 ins_cost(INSN_COST * 19);
9591 format %{ "udivw $dst, $src1, $src2" %}
9592
9593 ins_encode %{
9594 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9595 %}
9596
9597 ins_pipe(idiv_reg_reg);
9598 %}
9599
9600 // Unsigned Long Divide
9601
9602 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9603 match(Set dst (UDivL src1 src2));
9604
9605 ins_cost(INSN_COST * 35);
9606 format %{ "udiv $dst, $src1, $src2" %}
9607
9608 ins_encode %{
9609 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9610 %}
9611
9612 ins_pipe(ldiv_reg_reg);
9613 %}
9614
9615 // Unsigned Integer Remainder
9616
9617 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9618 match(Set dst (UModI src1 src2));
9619
9620 ins_cost(INSN_COST * 22);
9621 format %{ "udivw rscratch1, $src1, $src2\n\t"
9622 "msubw $dst, rscratch1, $src2, $src1" %}
9623
9624 ins_encode %{
9625 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9626 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9627 %}
9628
9629 ins_pipe(idiv_reg_reg);
9630 %}
9631
9632 // Unsigned Long Remainder
9633
9634 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9635 match(Set dst (UModL src1 src2));
9636
9637 ins_cost(INSN_COST * 38);
9638 format %{ "udiv rscratch1, $src1, $src2\n"
9639 "msub $dst, rscratch1, $src2, $src1" %}
9640
9641 ins_encode %{
9642 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9643 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9644 %}
9645
9646 ins_pipe(ldiv_reg_reg);
9647 %}
9648
9649 // Integer Shifts
9650
9651 // Shift Left Register
9652 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9653 match(Set dst (LShiftI src1 src2));
9654
9655 ins_cost(INSN_COST * 2);
9656 format %{ "lslvw $dst, $src1, $src2" %}
9657
9658 ins_encode %{
9659 __ lslvw(as_Register($dst$$reg),
9660 as_Register($src1$$reg),
9661 as_Register($src2$$reg));
9662 %}
9663
9664 ins_pipe(ialu_reg_reg_vshift);
9665 %}
9666
9667 // Shift Left Immediate
9668 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9669 match(Set dst (LShiftI src1 src2));
9670
9671 ins_cost(INSN_COST);
9672 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9673
9674 ins_encode %{
9675 __ lslw(as_Register($dst$$reg),
9676 as_Register($src1$$reg),
9677 $src2$$constant & 0x1f);
9678 %}
9679
9680 ins_pipe(ialu_reg_shift);
9681 %}
9682
9683 // Shift Right Logical Register
9684 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9685 match(Set dst (URShiftI src1 src2));
9686
9687 ins_cost(INSN_COST * 2);
9688 format %{ "lsrvw $dst, $src1, $src2" %}
9689
9690 ins_encode %{
9691 __ lsrvw(as_Register($dst$$reg),
9692 as_Register($src1$$reg),
9693 as_Register($src2$$reg));
9694 %}
9695
9696 ins_pipe(ialu_reg_reg_vshift);
9697 %}
9698
9699 // Shift Right Logical Immediate
9700 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9701 match(Set dst (URShiftI src1 src2));
9702
9703 ins_cost(INSN_COST);
9704 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9705
9706 ins_encode %{
9707 __ lsrw(as_Register($dst$$reg),
9708 as_Register($src1$$reg),
9709 $src2$$constant & 0x1f);
9710 %}
9711
9712 ins_pipe(ialu_reg_shift);
9713 %}
9714
9715 // Shift Right Arithmetic Register
9716 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9717 match(Set dst (RShiftI src1 src2));
9718
9719 ins_cost(INSN_COST * 2);
9720 format %{ "asrvw $dst, $src1, $src2" %}
9721
9722 ins_encode %{
9723 __ asrvw(as_Register($dst$$reg),
9724 as_Register($src1$$reg),
9725 as_Register($src2$$reg));
9726 %}
9727
9728 ins_pipe(ialu_reg_reg_vshift);
9729 %}
9730
9731 // Shift Right Arithmetic Immediate
9732 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9733 match(Set dst (RShiftI src1 src2));
9734
9735 ins_cost(INSN_COST);
9736 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9737
9738 ins_encode %{
9739 __ asrw(as_Register($dst$$reg),
9740 as_Register($src1$$reg),
9741 $src2$$constant & 0x1f);
9742 %}
9743
9744 ins_pipe(ialu_reg_shift);
9745 %}
9746
9747 // Combined Int Mask and Right Shift (using UBFM)
9748 // TODO
9749
9750 // Long Shifts
9751
9752 // Shift Left Register
9753 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9754 match(Set dst (LShiftL src1 src2));
9755
9756 ins_cost(INSN_COST * 2);
9757 format %{ "lslv $dst, $src1, $src2" %}
9758
9759 ins_encode %{
9760 __ lslv(as_Register($dst$$reg),
9761 as_Register($src1$$reg),
9762 as_Register($src2$$reg));
9763 %}
9764
9765 ins_pipe(ialu_reg_reg_vshift);
9766 %}
9767
9768 // Shift Left Immediate
9769 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9770 match(Set dst (LShiftL src1 src2));
9771
9772 ins_cost(INSN_COST);
9773 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9774
9775 ins_encode %{
9776 __ lsl(as_Register($dst$$reg),
9777 as_Register($src1$$reg),
9778 $src2$$constant & 0x3f);
9779 %}
9780
9781 ins_pipe(ialu_reg_shift);
9782 %}
9783
9784 // Shift Right Logical Register
9785 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9786 match(Set dst (URShiftL src1 src2));
9787
9788 ins_cost(INSN_COST * 2);
9789 format %{ "lsrv $dst, $src1, $src2" %}
9790
9791 ins_encode %{
9792 __ lsrv(as_Register($dst$$reg),
9793 as_Register($src1$$reg),
9794 as_Register($src2$$reg));
9795 %}
9796
9797 ins_pipe(ialu_reg_reg_vshift);
9798 %}
9799
9800 // Shift Right Logical Immediate
9801 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9802 match(Set dst (URShiftL src1 src2));
9803
9804 ins_cost(INSN_COST);
9805 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9806
9807 ins_encode %{
9808 __ lsr(as_Register($dst$$reg),
9809 as_Register($src1$$reg),
9810 $src2$$constant & 0x3f);
9811 %}
9812
9813 ins_pipe(ialu_reg_shift);
9814 %}
9815
9816 // A special-case pattern for card table stores.
9817 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9818 match(Set dst (URShiftL (CastP2X src1) src2));
9819
9820 ins_cost(INSN_COST);
9821 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9822
9823 ins_encode %{
9824 __ lsr(as_Register($dst$$reg),
9825 as_Register($src1$$reg),
9826 $src2$$constant & 0x3f);
9827 %}
9828
9829 ins_pipe(ialu_reg_shift);
9830 %}
9831
9832 // Shift Right Arithmetic Register
9833 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9834 match(Set dst (RShiftL src1 src2));
9835
9836 ins_cost(INSN_COST * 2);
9837 format %{ "asrv $dst, $src1, $src2" %}
9838
9839 ins_encode %{
9840 __ asrv(as_Register($dst$$reg),
9841 as_Register($src1$$reg),
9842 as_Register($src2$$reg));
9843 %}
9844
9845 ins_pipe(ialu_reg_reg_vshift);
9846 %}
9847
9848 // Shift Right Arithmetic Immediate
9849 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9850 match(Set dst (RShiftL src1 src2));
9851
9852 ins_cost(INSN_COST);
9853 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9854
9855 ins_encode %{
9856 __ asr(as_Register($dst$$reg),
9857 as_Register($src1$$reg),
9858 $src2$$constant & 0x3f);
9859 %}
9860
9861 ins_pipe(ialu_reg_shift);
9862 %}
9863
9864 // BEGIN This section of the file is automatically generated. Do not edit --------------
9865 // This section is generated from aarch64_ad.m4
9866
9867 // This pattern is automatically generated from aarch64_ad.m4
9868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9869 instruct regL_not_reg(iRegLNoSp dst,
9870 iRegL src1, immL_M1 m1,
9871 rFlagsReg cr) %{
9872 match(Set dst (XorL src1 m1));
9873 ins_cost(INSN_COST);
9874 format %{ "eon $dst, $src1, zr" %}
9875
9876 ins_encode %{
9877 __ eon(as_Register($dst$$reg),
9878 as_Register($src1$$reg),
9879 zr,
9880 Assembler::LSL, 0);
9881 %}
9882
9883 ins_pipe(ialu_reg);
9884 %}
9885
9886 // This pattern is automatically generated from aarch64_ad.m4
9887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9888 instruct regI_not_reg(iRegINoSp dst,
9889 iRegIorL2I src1, immI_M1 m1,
9890 rFlagsReg cr) %{
9891 match(Set dst (XorI src1 m1));
9892 ins_cost(INSN_COST);
9893 format %{ "eonw $dst, $src1, zr" %}
9894
9895 ins_encode %{
9896 __ eonw(as_Register($dst$$reg),
9897 as_Register($src1$$reg),
9898 zr,
9899 Assembler::LSL, 0);
9900 %}
9901
9902 ins_pipe(ialu_reg);
9903 %}
9904
9905 // This pattern is automatically generated from aarch64_ad.m4
9906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9907 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9908 immI0 zero, iRegIorL2I src1, immI src2) %{
9909 match(Set dst (SubI zero (URShiftI src1 src2)));
9910
9911 ins_cost(1.9 * INSN_COST);
9912 format %{ "negw $dst, $src1, LSR $src2" %}
9913
9914 ins_encode %{
9915 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9916 Assembler::LSR, $src2$$constant & 0x1f);
9917 %}
9918
9919 ins_pipe(ialu_reg_shift);
9920 %}
9921
9922 // This pattern is automatically generated from aarch64_ad.m4
9923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9924 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9925 immI0 zero, iRegIorL2I src1, immI src2) %{
9926 match(Set dst (SubI zero (RShiftI src1 src2)));
9927
9928 ins_cost(1.9 * INSN_COST);
9929 format %{ "negw $dst, $src1, ASR $src2" %}
9930
9931 ins_encode %{
9932 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9933 Assembler::ASR, $src2$$constant & 0x1f);
9934 %}
9935
9936 ins_pipe(ialu_reg_shift);
9937 %}
9938
9939 // This pattern is automatically generated from aarch64_ad.m4
9940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9941 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9942 immI0 zero, iRegIorL2I src1, immI src2) %{
9943 match(Set dst (SubI zero (LShiftI src1 src2)));
9944
9945 ins_cost(1.9 * INSN_COST);
9946 format %{ "negw $dst, $src1, LSL $src2" %}
9947
9948 ins_encode %{
9949 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9950 Assembler::LSL, $src2$$constant & 0x1f);
9951 %}
9952
9953 ins_pipe(ialu_reg_shift);
9954 %}
9955
9956 // This pattern is automatically generated from aarch64_ad.m4
9957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9958 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9959 immL0 zero, iRegL src1, immI src2) %{
9960 match(Set dst (SubL zero (URShiftL src1 src2)));
9961
9962 ins_cost(1.9 * INSN_COST);
9963 format %{ "neg $dst, $src1, LSR $src2" %}
9964
9965 ins_encode %{
9966 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9967 Assembler::LSR, $src2$$constant & 0x3f);
9968 %}
9969
9970 ins_pipe(ialu_reg_shift);
9971 %}
9972
9973 // This pattern is automatically generated from aarch64_ad.m4
9974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9975 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9976 immL0 zero, iRegL src1, immI src2) %{
9977 match(Set dst (SubL zero (RShiftL src1 src2)));
9978
9979 ins_cost(1.9 * INSN_COST);
9980 format %{ "neg $dst, $src1, ASR $src2" %}
9981
9982 ins_encode %{
9983 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9984 Assembler::ASR, $src2$$constant & 0x3f);
9985 %}
9986
9987 ins_pipe(ialu_reg_shift);
9988 %}
9989
9990 // This pattern is automatically generated from aarch64_ad.m4
9991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9992 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9993 immL0 zero, iRegL src1, immI src2) %{
9994 match(Set dst (SubL zero (LShiftL src1 src2)));
9995
9996 ins_cost(1.9 * INSN_COST);
9997 format %{ "neg $dst, $src1, LSL $src2" %}
9998
9999 ins_encode %{
10000 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10001 Assembler::LSL, $src2$$constant & 0x3f);
10002 %}
10003
10004 ins_pipe(ialu_reg_shift);
10005 %}
10006
10007 // This pattern is automatically generated from aarch64_ad.m4
10008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10009 instruct AndI_reg_not_reg(iRegINoSp dst,
10010 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10011 match(Set dst (AndI src1 (XorI src2 m1)));
10012 ins_cost(INSN_COST);
10013 format %{ "bicw $dst, $src1, $src2" %}
10014
10015 ins_encode %{
10016 __ bicw(as_Register($dst$$reg),
10017 as_Register($src1$$reg),
10018 as_Register($src2$$reg),
10019 Assembler::LSL, 0);
10020 %}
10021
10022 ins_pipe(ialu_reg_reg);
10023 %}
10024
10025 // This pattern is automatically generated from aarch64_ad.m4
10026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10027 instruct AndL_reg_not_reg(iRegLNoSp dst,
10028 iRegL src1, iRegL src2, immL_M1 m1) %{
10029 match(Set dst (AndL src1 (XorL src2 m1)));
10030 ins_cost(INSN_COST);
10031 format %{ "bic $dst, $src1, $src2" %}
10032
10033 ins_encode %{
10034 __ bic(as_Register($dst$$reg),
10035 as_Register($src1$$reg),
10036 as_Register($src2$$reg),
10037 Assembler::LSL, 0);
10038 %}
10039
10040 ins_pipe(ialu_reg_reg);
10041 %}
10042
10043 // This pattern is automatically generated from aarch64_ad.m4
10044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10045 instruct OrI_reg_not_reg(iRegINoSp dst,
10046 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10047 match(Set dst (OrI src1 (XorI src2 m1)));
10048 ins_cost(INSN_COST);
10049 format %{ "ornw $dst, $src1, $src2" %}
10050
10051 ins_encode %{
10052 __ ornw(as_Register($dst$$reg),
10053 as_Register($src1$$reg),
10054 as_Register($src2$$reg),
10055 Assembler::LSL, 0);
10056 %}
10057
10058 ins_pipe(ialu_reg_reg);
10059 %}
10060
10061 // This pattern is automatically generated from aarch64_ad.m4
10062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10063 instruct OrL_reg_not_reg(iRegLNoSp dst,
10064 iRegL src1, iRegL src2, immL_M1 m1) %{
10065 match(Set dst (OrL src1 (XorL src2 m1)));
10066 ins_cost(INSN_COST);
10067 format %{ "orn $dst, $src1, $src2" %}
10068
10069 ins_encode %{
10070 __ orn(as_Register($dst$$reg),
10071 as_Register($src1$$reg),
10072 as_Register($src2$$reg),
10073 Assembler::LSL, 0);
10074 %}
10075
10076 ins_pipe(ialu_reg_reg);
10077 %}
10078
10079 // This pattern is automatically generated from aarch64_ad.m4
10080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10081 instruct XorI_reg_not_reg(iRegINoSp dst,
10082 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10083 match(Set dst (XorI m1 (XorI src2 src1)));
10084 ins_cost(INSN_COST);
10085 format %{ "eonw $dst, $src1, $src2" %}
10086
10087 ins_encode %{
10088 __ eonw(as_Register($dst$$reg),
10089 as_Register($src1$$reg),
10090 as_Register($src2$$reg),
10091 Assembler::LSL, 0);
10092 %}
10093
10094 ins_pipe(ialu_reg_reg);
10095 %}
10096
10097 // This pattern is automatically generated from aarch64_ad.m4
10098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10099 instruct XorL_reg_not_reg(iRegLNoSp dst,
10100 iRegL src1, iRegL src2, immL_M1 m1) %{
10101 match(Set dst (XorL m1 (XorL src2 src1)));
10102 ins_cost(INSN_COST);
10103 format %{ "eon $dst, $src1, $src2" %}
10104
10105 ins_encode %{
10106 __ eon(as_Register($dst$$reg),
10107 as_Register($src1$$reg),
10108 as_Register($src2$$reg),
10109 Assembler::LSL, 0);
10110 %}
10111
10112 ins_pipe(ialu_reg_reg);
10113 %}
10114
10115 // This pattern is automatically generated from aarch64_ad.m4
10116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10117 // val & (-1 ^ (val >>> shift)) ==> bicw
10118 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10119 iRegIorL2I src1, iRegIorL2I src2,
10120 immI src3, immI_M1 src4) %{
10121 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10122 ins_cost(1.9 * INSN_COST);
10123 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10124
10125 ins_encode %{
10126 __ bicw(as_Register($dst$$reg),
10127 as_Register($src1$$reg),
10128 as_Register($src2$$reg),
10129 Assembler::LSR,
10130 $src3$$constant & 0x1f);
10131 %}
10132
10133 ins_pipe(ialu_reg_reg_shift);
10134 %}
10135
10136 // This pattern is automatically generated from aarch64_ad.m4
10137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10138 // val & (-1 ^ (val >>> shift)) ==> bic
10139 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10140 iRegL src1, iRegL src2,
10141 immI src3, immL_M1 src4) %{
10142 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10143 ins_cost(1.9 * INSN_COST);
10144 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10145
10146 ins_encode %{
10147 __ bic(as_Register($dst$$reg),
10148 as_Register($src1$$reg),
10149 as_Register($src2$$reg),
10150 Assembler::LSR,
10151 $src3$$constant & 0x3f);
10152 %}
10153
10154 ins_pipe(ialu_reg_reg_shift);
10155 %}
10156
10157 // This pattern is automatically generated from aarch64_ad.m4
10158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10159 // val & (-1 ^ (val >> shift)) ==> bicw
10160 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10161 iRegIorL2I src1, iRegIorL2I src2,
10162 immI src3, immI_M1 src4) %{
10163 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10164 ins_cost(1.9 * INSN_COST);
10165 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10166
10167 ins_encode %{
10168 __ bicw(as_Register($dst$$reg),
10169 as_Register($src1$$reg),
10170 as_Register($src2$$reg),
10171 Assembler::ASR,
10172 $src3$$constant & 0x1f);
10173 %}
10174
10175 ins_pipe(ialu_reg_reg_shift);
10176 %}
10177
10178 // This pattern is automatically generated from aarch64_ad.m4
10179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10180 // val & (-1 ^ (val >> shift)) ==> bic
10181 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10182 iRegL src1, iRegL src2,
10183 immI src3, immL_M1 src4) %{
10184 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10185 ins_cost(1.9 * INSN_COST);
10186 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10187
10188 ins_encode %{
10189 __ bic(as_Register($dst$$reg),
10190 as_Register($src1$$reg),
10191 as_Register($src2$$reg),
10192 Assembler::ASR,
10193 $src3$$constant & 0x3f);
10194 %}
10195
10196 ins_pipe(ialu_reg_reg_shift);
10197 %}
10198
10199 // This pattern is automatically generated from aarch64_ad.m4
10200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10201 // val & (-1 ^ (val ror shift)) ==> bicw
10202 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10203 iRegIorL2I src1, iRegIorL2I src2,
10204 immI src3, immI_M1 src4) %{
10205 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10206 ins_cost(1.9 * INSN_COST);
10207 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10208
10209 ins_encode %{
10210 __ bicw(as_Register($dst$$reg),
10211 as_Register($src1$$reg),
10212 as_Register($src2$$reg),
10213 Assembler::ROR,
10214 $src3$$constant & 0x1f);
10215 %}
10216
10217 ins_pipe(ialu_reg_reg_shift);
10218 %}
10219
10220 // This pattern is automatically generated from aarch64_ad.m4
10221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10222 // val & (-1 ^ (val ror shift)) ==> bic
10223 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10224 iRegL src1, iRegL src2,
10225 immI src3, immL_M1 src4) %{
10226 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10227 ins_cost(1.9 * INSN_COST);
10228 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10229
10230 ins_encode %{
10231 __ bic(as_Register($dst$$reg),
10232 as_Register($src1$$reg),
10233 as_Register($src2$$reg),
10234 Assembler::ROR,
10235 $src3$$constant & 0x3f);
10236 %}
10237
10238 ins_pipe(ialu_reg_reg_shift);
10239 %}
10240
10241 // This pattern is automatically generated from aarch64_ad.m4
10242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10243 // val & (-1 ^ (val << shift)) ==> bicw
10244 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10245 iRegIorL2I src1, iRegIorL2I src2,
10246 immI src3, immI_M1 src4) %{
10247 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10248 ins_cost(1.9 * INSN_COST);
10249 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10250
10251 ins_encode %{
10252 __ bicw(as_Register($dst$$reg),
10253 as_Register($src1$$reg),
10254 as_Register($src2$$reg),
10255 Assembler::LSL,
10256 $src3$$constant & 0x1f);
10257 %}
10258
10259 ins_pipe(ialu_reg_reg_shift);
10260 %}
10261
10262 // This pattern is automatically generated from aarch64_ad.m4
10263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10264 // val & (-1 ^ (val << shift)) ==> bic
10265 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10266 iRegL src1, iRegL src2,
10267 immI src3, immL_M1 src4) %{
10268 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10269 ins_cost(1.9 * INSN_COST);
10270 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10271
10272 ins_encode %{
10273 __ bic(as_Register($dst$$reg),
10274 as_Register($src1$$reg),
10275 as_Register($src2$$reg),
10276 Assembler::LSL,
10277 $src3$$constant & 0x3f);
10278 %}
10279
10280 ins_pipe(ialu_reg_reg_shift);
10281 %}
10282
10283 // This pattern is automatically generated from aarch64_ad.m4
10284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10285 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10286 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10287 iRegIorL2I src1, iRegIorL2I src2,
10288 immI src3, immI_M1 src4) %{
10289 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10290 ins_cost(1.9 * INSN_COST);
10291 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10292
10293 ins_encode %{
10294 __ eonw(as_Register($dst$$reg),
10295 as_Register($src1$$reg),
10296 as_Register($src2$$reg),
10297 Assembler::LSR,
10298 $src3$$constant & 0x1f);
10299 %}
10300
10301 ins_pipe(ialu_reg_reg_shift);
10302 %}
10303
10304 // This pattern is automatically generated from aarch64_ad.m4
10305 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10306 // val ^ (-1 ^ (val >>> shift)) ==> eon
10307 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10308 iRegL src1, iRegL src2,
10309 immI src3, immL_M1 src4) %{
10310 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10311 ins_cost(1.9 * INSN_COST);
10312 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10313
10314 ins_encode %{
10315 __ eon(as_Register($dst$$reg),
10316 as_Register($src1$$reg),
10317 as_Register($src2$$reg),
10318 Assembler::LSR,
10319 $src3$$constant & 0x3f);
10320 %}
10321
10322 ins_pipe(ialu_reg_reg_shift);
10323 %}
10324
10325 // This pattern is automatically generated from aarch64_ad.m4
10326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10327 // val ^ (-1 ^ (val >> shift)) ==> eonw
10328 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10329 iRegIorL2I src1, iRegIorL2I src2,
10330 immI src3, immI_M1 src4) %{
10331 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10332 ins_cost(1.9 * INSN_COST);
10333 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10334
10335 ins_encode %{
10336 __ eonw(as_Register($dst$$reg),
10337 as_Register($src1$$reg),
10338 as_Register($src2$$reg),
10339 Assembler::ASR,
10340 $src3$$constant & 0x1f);
10341 %}
10342
10343 ins_pipe(ialu_reg_reg_shift);
10344 %}
10345
10346 // This pattern is automatically generated from aarch64_ad.m4
10347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10348 // val ^ (-1 ^ (val >> shift)) ==> eon
10349 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10350 iRegL src1, iRegL src2,
10351 immI src3, immL_M1 src4) %{
10352 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10353 ins_cost(1.9 * INSN_COST);
10354 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10355
10356 ins_encode %{
10357 __ eon(as_Register($dst$$reg),
10358 as_Register($src1$$reg),
10359 as_Register($src2$$reg),
10360 Assembler::ASR,
10361 $src3$$constant & 0x3f);
10362 %}
10363
10364 ins_pipe(ialu_reg_reg_shift);
10365 %}
10366
10367 // This pattern is automatically generated from aarch64_ad.m4
10368 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10369 // val ^ (-1 ^ (val ror shift)) ==> eonw
10370 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10371 iRegIorL2I src1, iRegIorL2I src2,
10372 immI src3, immI_M1 src4) %{
10373 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10374 ins_cost(1.9 * INSN_COST);
10375 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10376
10377 ins_encode %{
10378 __ eonw(as_Register($dst$$reg),
10379 as_Register($src1$$reg),
10380 as_Register($src2$$reg),
10381 Assembler::ROR,
10382 $src3$$constant & 0x1f);
10383 %}
10384
10385 ins_pipe(ialu_reg_reg_shift);
10386 %}
10387
10388 // This pattern is automatically generated from aarch64_ad.m4
10389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10390 // val ^ (-1 ^ (val ror shift)) ==> eon
10391 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10392 iRegL src1, iRegL src2,
10393 immI src3, immL_M1 src4) %{
10394 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10395 ins_cost(1.9 * INSN_COST);
10396 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10397
10398 ins_encode %{
10399 __ eon(as_Register($dst$$reg),
10400 as_Register($src1$$reg),
10401 as_Register($src2$$reg),
10402 Assembler::ROR,
10403 $src3$$constant & 0x3f);
10404 %}
10405
10406 ins_pipe(ialu_reg_reg_shift);
10407 %}
10408
10409 // This pattern is automatically generated from aarch64_ad.m4
10410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10411 // val ^ (-1 ^ (val << shift)) ==> eonw
10412 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10413 iRegIorL2I src1, iRegIorL2I src2,
10414 immI src3, immI_M1 src4) %{
10415 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10416 ins_cost(1.9 * INSN_COST);
10417 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10418
10419 ins_encode %{
10420 __ eonw(as_Register($dst$$reg),
10421 as_Register($src1$$reg),
10422 as_Register($src2$$reg),
10423 Assembler::LSL,
10424 $src3$$constant & 0x1f);
10425 %}
10426
10427 ins_pipe(ialu_reg_reg_shift);
10428 %}
10429
10430 // This pattern is automatically generated from aarch64_ad.m4
10431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10432 // val ^ (-1 ^ (val << shift)) ==> eon
10433 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10434 iRegL src1, iRegL src2,
10435 immI src3, immL_M1 src4) %{
10436 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10437 ins_cost(1.9 * INSN_COST);
10438 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10439
10440 ins_encode %{
10441 __ eon(as_Register($dst$$reg),
10442 as_Register($src1$$reg),
10443 as_Register($src2$$reg),
10444 Assembler::LSL,
10445 $src3$$constant & 0x3f);
10446 %}
10447
10448 ins_pipe(ialu_reg_reg_shift);
10449 %}
10450
10451 // This pattern is automatically generated from aarch64_ad.m4
10452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10453 // val | (-1 ^ (val >>> shift)) ==> ornw
10454 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10455 iRegIorL2I src1, iRegIorL2I src2,
10456 immI src3, immI_M1 src4) %{
10457 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10458 ins_cost(1.9 * INSN_COST);
10459 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10460
10461 ins_encode %{
10462 __ ornw(as_Register($dst$$reg),
10463 as_Register($src1$$reg),
10464 as_Register($src2$$reg),
10465 Assembler::LSR,
10466 $src3$$constant & 0x1f);
10467 %}
10468
10469 ins_pipe(ialu_reg_reg_shift);
10470 %}
10471
10472 // This pattern is automatically generated from aarch64_ad.m4
10473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10474 // val | (-1 ^ (val >>> shift)) ==> orn
10475 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10476 iRegL src1, iRegL src2,
10477 immI src3, immL_M1 src4) %{
10478 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10479 ins_cost(1.9 * INSN_COST);
10480 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10481
10482 ins_encode %{
10483 __ orn(as_Register($dst$$reg),
10484 as_Register($src1$$reg),
10485 as_Register($src2$$reg),
10486 Assembler::LSR,
10487 $src3$$constant & 0x3f);
10488 %}
10489
10490 ins_pipe(ialu_reg_reg_shift);
10491 %}
10492
10493 // This pattern is automatically generated from aarch64_ad.m4
10494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10495 // val | (-1 ^ (val >> shift)) ==> ornw
10496 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10497 iRegIorL2I src1, iRegIorL2I src2,
10498 immI src3, immI_M1 src4) %{
10499 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10500 ins_cost(1.9 * INSN_COST);
10501 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10502
10503 ins_encode %{
10504 __ ornw(as_Register($dst$$reg),
10505 as_Register($src1$$reg),
10506 as_Register($src2$$reg),
10507 Assembler::ASR,
10508 $src3$$constant & 0x1f);
10509 %}
10510
10511 ins_pipe(ialu_reg_reg_shift);
10512 %}
10513
10514 // This pattern is automatically generated from aarch64_ad.m4
10515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10516 // val | (-1 ^ (val >> shift)) ==> orn
10517 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10518 iRegL src1, iRegL src2,
10519 immI src3, immL_M1 src4) %{
10520 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10521 ins_cost(1.9 * INSN_COST);
10522 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10523
10524 ins_encode %{
10525 __ orn(as_Register($dst$$reg),
10526 as_Register($src1$$reg),
10527 as_Register($src2$$reg),
10528 Assembler::ASR,
10529 $src3$$constant & 0x3f);
10530 %}
10531
10532 ins_pipe(ialu_reg_reg_shift);
10533 %}
10534
10535 // This pattern is automatically generated from aarch64_ad.m4
10536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10537 // val | (-1 ^ (val ror shift)) ==> ornw
10538 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10539 iRegIorL2I src1, iRegIorL2I src2,
10540 immI src3, immI_M1 src4) %{
10541 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10542 ins_cost(1.9 * INSN_COST);
10543 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10544
10545 ins_encode %{
10546 __ ornw(as_Register($dst$$reg),
10547 as_Register($src1$$reg),
10548 as_Register($src2$$reg),
10549 Assembler::ROR,
10550 $src3$$constant & 0x1f);
10551 %}
10552
10553 ins_pipe(ialu_reg_reg_shift);
10554 %}
10555
10556 // This pattern is automatically generated from aarch64_ad.m4
10557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10558 // val | (-1 ^ (val ror shift)) ==> orn
10559 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10560 iRegL src1, iRegL src2,
10561 immI src3, immL_M1 src4) %{
10562 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10563 ins_cost(1.9 * INSN_COST);
10564 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10565
10566 ins_encode %{
10567 __ orn(as_Register($dst$$reg),
10568 as_Register($src1$$reg),
10569 as_Register($src2$$reg),
10570 Assembler::ROR,
10571 $src3$$constant & 0x3f);
10572 %}
10573
10574 ins_pipe(ialu_reg_reg_shift);
10575 %}
10576
10577 // This pattern is automatically generated from aarch64_ad.m4
10578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10579 // val | (-1 ^ (val << shift)) ==> ornw
10580 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10581 iRegIorL2I src1, iRegIorL2I src2,
10582 immI src3, immI_M1 src4) %{
10583 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10584 ins_cost(1.9 * INSN_COST);
10585 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10586
10587 ins_encode %{
10588 __ ornw(as_Register($dst$$reg),
10589 as_Register($src1$$reg),
10590 as_Register($src2$$reg),
10591 Assembler::LSL,
10592 $src3$$constant & 0x1f);
10593 %}
10594
10595 ins_pipe(ialu_reg_reg_shift);
10596 %}
10597
10598 // This pattern is automatically generated from aarch64_ad.m4
10599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10600 // val | (-1 ^ (val << shift)) ==> orn
10601 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10602 iRegL src1, iRegL src2,
10603 immI src3, immL_M1 src4) %{
10604 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10605 ins_cost(1.9 * INSN_COST);
10606 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10607
10608 ins_encode %{
10609 __ orn(as_Register($dst$$reg),
10610 as_Register($src1$$reg),
10611 as_Register($src2$$reg),
10612 Assembler::LSL,
10613 $src3$$constant & 0x3f);
10614 %}
10615
10616 ins_pipe(ialu_reg_reg_shift);
10617 %}
10618
10619 // This pattern is automatically generated from aarch64_ad.m4
10620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10621 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10622 iRegIorL2I src1, iRegIorL2I src2,
10623 immI src3) %{
10624 match(Set dst (AndI src1 (URShiftI src2 src3)));
10625
10626 ins_cost(1.9 * INSN_COST);
10627 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10628
10629 ins_encode %{
10630 __ andw(as_Register($dst$$reg),
10631 as_Register($src1$$reg),
10632 as_Register($src2$$reg),
10633 Assembler::LSR,
10634 $src3$$constant & 0x1f);
10635 %}
10636
10637 ins_pipe(ialu_reg_reg_shift);
10638 %}
10639
10640 // This pattern is automatically generated from aarch64_ad.m4
10641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10642 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10643 iRegL src1, iRegL src2,
10644 immI src3) %{
10645 match(Set dst (AndL src1 (URShiftL src2 src3)));
10646
10647 ins_cost(1.9 * INSN_COST);
10648 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10649
10650 ins_encode %{
10651 __ andr(as_Register($dst$$reg),
10652 as_Register($src1$$reg),
10653 as_Register($src2$$reg),
10654 Assembler::LSR,
10655 $src3$$constant & 0x3f);
10656 %}
10657
10658 ins_pipe(ialu_reg_reg_shift);
10659 %}
10660
10661 // This pattern is automatically generated from aarch64_ad.m4
10662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10663 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10664 iRegIorL2I src1, iRegIorL2I src2,
10665 immI src3) %{
10666 match(Set dst (AndI src1 (RShiftI src2 src3)));
10667
10668 ins_cost(1.9 * INSN_COST);
10669 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10670
10671 ins_encode %{
10672 __ andw(as_Register($dst$$reg),
10673 as_Register($src1$$reg),
10674 as_Register($src2$$reg),
10675 Assembler::ASR,
10676 $src3$$constant & 0x1f);
10677 %}
10678
10679 ins_pipe(ialu_reg_reg_shift);
10680 %}
10681
10682 // This pattern is automatically generated from aarch64_ad.m4
10683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10684 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10685 iRegL src1, iRegL src2,
10686 immI src3) %{
10687 match(Set dst (AndL src1 (RShiftL src2 src3)));
10688
10689 ins_cost(1.9 * INSN_COST);
10690 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10691
10692 ins_encode %{
10693 __ andr(as_Register($dst$$reg),
10694 as_Register($src1$$reg),
10695 as_Register($src2$$reg),
10696 Assembler::ASR,
10697 $src3$$constant & 0x3f);
10698 %}
10699
10700 ins_pipe(ialu_reg_reg_shift);
10701 %}
10702
10703 // This pattern is automatically generated from aarch64_ad.m4
10704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10705 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10706 iRegIorL2I src1, iRegIorL2I src2,
10707 immI src3) %{
10708 match(Set dst (AndI src1 (LShiftI src2 src3)));
10709
10710 ins_cost(1.9 * INSN_COST);
10711 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10712
10713 ins_encode %{
10714 __ andw(as_Register($dst$$reg),
10715 as_Register($src1$$reg),
10716 as_Register($src2$$reg),
10717 Assembler::LSL,
10718 $src3$$constant & 0x1f);
10719 %}
10720
10721 ins_pipe(ialu_reg_reg_shift);
10722 %}
10723
10724 // This pattern is automatically generated from aarch64_ad.m4
10725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10726 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10727 iRegL src1, iRegL src2,
10728 immI src3) %{
10729 match(Set dst (AndL src1 (LShiftL src2 src3)));
10730
10731 ins_cost(1.9 * INSN_COST);
10732 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10733
10734 ins_encode %{
10735 __ andr(as_Register($dst$$reg),
10736 as_Register($src1$$reg),
10737 as_Register($src2$$reg),
10738 Assembler::LSL,
10739 $src3$$constant & 0x3f);
10740 %}
10741
10742 ins_pipe(ialu_reg_reg_shift);
10743 %}
10744
10745 // This pattern is automatically generated from aarch64_ad.m4
10746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10747 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10748 iRegIorL2I src1, iRegIorL2I src2,
10749 immI src3) %{
10750 match(Set dst (AndI src1 (RotateRight src2 src3)));
10751
10752 ins_cost(1.9 * INSN_COST);
10753 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10754
10755 ins_encode %{
10756 __ andw(as_Register($dst$$reg),
10757 as_Register($src1$$reg),
10758 as_Register($src2$$reg),
10759 Assembler::ROR,
10760 $src3$$constant & 0x1f);
10761 %}
10762
10763 ins_pipe(ialu_reg_reg_shift);
10764 %}
10765
10766 // This pattern is automatically generated from aarch64_ad.m4
10767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10768 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10769 iRegL src1, iRegL src2,
10770 immI src3) %{
10771 match(Set dst (AndL src1 (RotateRight src2 src3)));
10772
10773 ins_cost(1.9 * INSN_COST);
10774 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10775
10776 ins_encode %{
10777 __ andr(as_Register($dst$$reg),
10778 as_Register($src1$$reg),
10779 as_Register($src2$$reg),
10780 Assembler::ROR,
10781 $src3$$constant & 0x3f);
10782 %}
10783
10784 ins_pipe(ialu_reg_reg_shift);
10785 %}
10786
10787 // This pattern is automatically generated from aarch64_ad.m4
10788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10789 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10790 iRegIorL2I src1, iRegIorL2I src2,
10791 immI src3) %{
10792 match(Set dst (XorI src1 (URShiftI src2 src3)));
10793
10794 ins_cost(1.9 * INSN_COST);
10795 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10796
10797 ins_encode %{
10798 __ eorw(as_Register($dst$$reg),
10799 as_Register($src1$$reg),
10800 as_Register($src2$$reg),
10801 Assembler::LSR,
10802 $src3$$constant & 0x1f);
10803 %}
10804
10805 ins_pipe(ialu_reg_reg_shift);
10806 %}
10807
10808 // This pattern is automatically generated from aarch64_ad.m4
10809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10810 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10811 iRegL src1, iRegL src2,
10812 immI src3) %{
10813 match(Set dst (XorL src1 (URShiftL src2 src3)));
10814
10815 ins_cost(1.9 * INSN_COST);
10816 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10817
10818 ins_encode %{
10819 __ eor(as_Register($dst$$reg),
10820 as_Register($src1$$reg),
10821 as_Register($src2$$reg),
10822 Assembler::LSR,
10823 $src3$$constant & 0x3f);
10824 %}
10825
10826 ins_pipe(ialu_reg_reg_shift);
10827 %}
10828
10829 // This pattern is automatically generated from aarch64_ad.m4
10830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10831 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10832 iRegIorL2I src1, iRegIorL2I src2,
10833 immI src3) %{
10834 match(Set dst (XorI src1 (RShiftI src2 src3)));
10835
10836 ins_cost(1.9 * INSN_COST);
10837 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10838
10839 ins_encode %{
10840 __ eorw(as_Register($dst$$reg),
10841 as_Register($src1$$reg),
10842 as_Register($src2$$reg),
10843 Assembler::ASR,
10844 $src3$$constant & 0x1f);
10845 %}
10846
10847 ins_pipe(ialu_reg_reg_shift);
10848 %}
10849
10850 // This pattern is automatically generated from aarch64_ad.m4
10851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10852 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10853 iRegL src1, iRegL src2,
10854 immI src3) %{
10855 match(Set dst (XorL src1 (RShiftL src2 src3)));
10856
10857 ins_cost(1.9 * INSN_COST);
10858 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10859
10860 ins_encode %{
10861 __ eor(as_Register($dst$$reg),
10862 as_Register($src1$$reg),
10863 as_Register($src2$$reg),
10864 Assembler::ASR,
10865 $src3$$constant & 0x3f);
10866 %}
10867
10868 ins_pipe(ialu_reg_reg_shift);
10869 %}
10870
10871 // This pattern is automatically generated from aarch64_ad.m4
10872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10873 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10874 iRegIorL2I src1, iRegIorL2I src2,
10875 immI src3) %{
10876 match(Set dst (XorI src1 (LShiftI src2 src3)));
10877
10878 ins_cost(1.9 * INSN_COST);
10879 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10880
10881 ins_encode %{
10882 __ eorw(as_Register($dst$$reg),
10883 as_Register($src1$$reg),
10884 as_Register($src2$$reg),
10885 Assembler::LSL,
10886 $src3$$constant & 0x1f);
10887 %}
10888
10889 ins_pipe(ialu_reg_reg_shift);
10890 %}
10891
10892 // This pattern is automatically generated from aarch64_ad.m4
10893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10894 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10895 iRegL src1, iRegL src2,
10896 immI src3) %{
10897 match(Set dst (XorL src1 (LShiftL src2 src3)));
10898
10899 ins_cost(1.9 * INSN_COST);
10900 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10901
10902 ins_encode %{
10903 __ eor(as_Register($dst$$reg),
10904 as_Register($src1$$reg),
10905 as_Register($src2$$reg),
10906 Assembler::LSL,
10907 $src3$$constant & 0x3f);
10908 %}
10909
10910 ins_pipe(ialu_reg_reg_shift);
10911 %}
10912
10913 // This pattern is automatically generated from aarch64_ad.m4
10914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10915 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10916 iRegIorL2I src1, iRegIorL2I src2,
10917 immI src3) %{
10918 match(Set dst (XorI src1 (RotateRight src2 src3)));
10919
10920 ins_cost(1.9 * INSN_COST);
10921 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10922
10923 ins_encode %{
10924 __ eorw(as_Register($dst$$reg),
10925 as_Register($src1$$reg),
10926 as_Register($src2$$reg),
10927 Assembler::ROR,
10928 $src3$$constant & 0x1f);
10929 %}
10930
10931 ins_pipe(ialu_reg_reg_shift);
10932 %}
10933
10934 // This pattern is automatically generated from aarch64_ad.m4
10935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10936 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10937 iRegL src1, iRegL src2,
10938 immI src3) %{
10939 match(Set dst (XorL src1 (RotateRight src2 src3)));
10940
10941 ins_cost(1.9 * INSN_COST);
10942 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10943
10944 ins_encode %{
10945 __ eor(as_Register($dst$$reg),
10946 as_Register($src1$$reg),
10947 as_Register($src2$$reg),
10948 Assembler::ROR,
10949 $src3$$constant & 0x3f);
10950 %}
10951
10952 ins_pipe(ialu_reg_reg_shift);
10953 %}
10954
10955 // This pattern is automatically generated from aarch64_ad.m4
10956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10957 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10958 iRegIorL2I src1, iRegIorL2I src2,
10959 immI src3) %{
10960 match(Set dst (OrI src1 (URShiftI src2 src3)));
10961
10962 ins_cost(1.9 * INSN_COST);
10963 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10964
10965 ins_encode %{
10966 __ orrw(as_Register($dst$$reg),
10967 as_Register($src1$$reg),
10968 as_Register($src2$$reg),
10969 Assembler::LSR,
10970 $src3$$constant & 0x1f);
10971 %}
10972
10973 ins_pipe(ialu_reg_reg_shift);
10974 %}
10975
10976 // This pattern is automatically generated from aarch64_ad.m4
10977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10978 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10979 iRegL src1, iRegL src2,
10980 immI src3) %{
10981 match(Set dst (OrL src1 (URShiftL src2 src3)));
10982
10983 ins_cost(1.9 * INSN_COST);
10984 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10985
10986 ins_encode %{
10987 __ orr(as_Register($dst$$reg),
10988 as_Register($src1$$reg),
10989 as_Register($src2$$reg),
10990 Assembler::LSR,
10991 $src3$$constant & 0x3f);
10992 %}
10993
10994 ins_pipe(ialu_reg_reg_shift);
10995 %}
10996
10997 // This pattern is automatically generated from aarch64_ad.m4
10998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10999 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11000 iRegIorL2I src1, iRegIorL2I src2,
11001 immI src3) %{
11002 match(Set dst (OrI src1 (RShiftI src2 src3)));
11003
11004 ins_cost(1.9 * INSN_COST);
11005 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11006
11007 ins_encode %{
11008 __ orrw(as_Register($dst$$reg),
11009 as_Register($src1$$reg),
11010 as_Register($src2$$reg),
11011 Assembler::ASR,
11012 $src3$$constant & 0x1f);
11013 %}
11014
11015 ins_pipe(ialu_reg_reg_shift);
11016 %}
11017
11018 // This pattern is automatically generated from aarch64_ad.m4
11019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11020 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11021 iRegL src1, iRegL src2,
11022 immI src3) %{
11023 match(Set dst (OrL src1 (RShiftL src2 src3)));
11024
11025 ins_cost(1.9 * INSN_COST);
11026 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11027
11028 ins_encode %{
11029 __ orr(as_Register($dst$$reg),
11030 as_Register($src1$$reg),
11031 as_Register($src2$$reg),
11032 Assembler::ASR,
11033 $src3$$constant & 0x3f);
11034 %}
11035
11036 ins_pipe(ialu_reg_reg_shift);
11037 %}
11038
11039 // This pattern is automatically generated from aarch64_ad.m4
11040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11041 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11042 iRegIorL2I src1, iRegIorL2I src2,
11043 immI src3) %{
11044 match(Set dst (OrI src1 (LShiftI src2 src3)));
11045
11046 ins_cost(1.9 * INSN_COST);
11047 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11048
11049 ins_encode %{
11050 __ orrw(as_Register($dst$$reg),
11051 as_Register($src1$$reg),
11052 as_Register($src2$$reg),
11053 Assembler::LSL,
11054 $src3$$constant & 0x1f);
11055 %}
11056
11057 ins_pipe(ialu_reg_reg_shift);
11058 %}
11059
11060 // This pattern is automatically generated from aarch64_ad.m4
11061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11062 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11063 iRegL src1, iRegL src2,
11064 immI src3) %{
11065 match(Set dst (OrL src1 (LShiftL src2 src3)));
11066
11067 ins_cost(1.9 * INSN_COST);
11068 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11069
11070 ins_encode %{
11071 __ orr(as_Register($dst$$reg),
11072 as_Register($src1$$reg),
11073 as_Register($src2$$reg),
11074 Assembler::LSL,
11075 $src3$$constant & 0x3f);
11076 %}
11077
11078 ins_pipe(ialu_reg_reg_shift);
11079 %}
11080
11081 // This pattern is automatically generated from aarch64_ad.m4
11082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11083 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11084 iRegIorL2I src1, iRegIorL2I src2,
11085 immI src3) %{
11086 match(Set dst (OrI src1 (RotateRight src2 src3)));
11087
11088 ins_cost(1.9 * INSN_COST);
11089 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11090
11091 ins_encode %{
11092 __ orrw(as_Register($dst$$reg),
11093 as_Register($src1$$reg),
11094 as_Register($src2$$reg),
11095 Assembler::ROR,
11096 $src3$$constant & 0x1f);
11097 %}
11098
11099 ins_pipe(ialu_reg_reg_shift);
11100 %}
11101
11102 // This pattern is automatically generated from aarch64_ad.m4
11103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11104 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11105 iRegL src1, iRegL src2,
11106 immI src3) %{
11107 match(Set dst (OrL src1 (RotateRight src2 src3)));
11108
11109 ins_cost(1.9 * INSN_COST);
11110 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11111
11112 ins_encode %{
11113 __ orr(as_Register($dst$$reg),
11114 as_Register($src1$$reg),
11115 as_Register($src2$$reg),
11116 Assembler::ROR,
11117 $src3$$constant & 0x3f);
11118 %}
11119
11120 ins_pipe(ialu_reg_reg_shift);
11121 %}
11122
11123 // This pattern is automatically generated from aarch64_ad.m4
11124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11125 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11126 iRegIorL2I src1, iRegIorL2I src2,
11127 immI src3) %{
11128 match(Set dst (AddI src1 (URShiftI src2 src3)));
11129
11130 ins_cost(1.9 * INSN_COST);
11131 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11132
11133 ins_encode %{
11134 __ addw(as_Register($dst$$reg),
11135 as_Register($src1$$reg),
11136 as_Register($src2$$reg),
11137 Assembler::LSR,
11138 $src3$$constant & 0x1f);
11139 %}
11140
11141 ins_pipe(ialu_reg_reg_shift);
11142 %}
11143
11144 // This pattern is automatically generated from aarch64_ad.m4
11145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11146 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11147 iRegL src1, iRegL src2,
11148 immI src3) %{
11149 match(Set dst (AddL src1 (URShiftL src2 src3)));
11150
11151 ins_cost(1.9 * INSN_COST);
11152 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11153
11154 ins_encode %{
11155 __ add(as_Register($dst$$reg),
11156 as_Register($src1$$reg),
11157 as_Register($src2$$reg),
11158 Assembler::LSR,
11159 $src3$$constant & 0x3f);
11160 %}
11161
11162 ins_pipe(ialu_reg_reg_shift);
11163 %}
11164
11165 // This pattern is automatically generated from aarch64_ad.m4
11166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11167 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11168 iRegIorL2I src1, iRegIorL2I src2,
11169 immI src3) %{
11170 match(Set dst (AddI src1 (RShiftI src2 src3)));
11171
11172 ins_cost(1.9 * INSN_COST);
11173 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11174
11175 ins_encode %{
11176 __ addw(as_Register($dst$$reg),
11177 as_Register($src1$$reg),
11178 as_Register($src2$$reg),
11179 Assembler::ASR,
11180 $src3$$constant & 0x1f);
11181 %}
11182
11183 ins_pipe(ialu_reg_reg_shift);
11184 %}
11185
11186 // This pattern is automatically generated from aarch64_ad.m4
11187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11188 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11189 iRegL src1, iRegL src2,
11190 immI src3) %{
11191 match(Set dst (AddL src1 (RShiftL src2 src3)));
11192
11193 ins_cost(1.9 * INSN_COST);
11194 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11195
11196 ins_encode %{
11197 __ add(as_Register($dst$$reg),
11198 as_Register($src1$$reg),
11199 as_Register($src2$$reg),
11200 Assembler::ASR,
11201 $src3$$constant & 0x3f);
11202 %}
11203
11204 ins_pipe(ialu_reg_reg_shift);
11205 %}
11206
11207 // This pattern is automatically generated from aarch64_ad.m4
11208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11209 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11210 iRegIorL2I src1, iRegIorL2I src2,
11211 immI src3) %{
11212 match(Set dst (AddI src1 (LShiftI src2 src3)));
11213
11214 ins_cost(1.9 * INSN_COST);
11215 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11216
11217 ins_encode %{
11218 __ addw(as_Register($dst$$reg),
11219 as_Register($src1$$reg),
11220 as_Register($src2$$reg),
11221 Assembler::LSL,
11222 $src3$$constant & 0x1f);
11223 %}
11224
11225 ins_pipe(ialu_reg_reg_shift);
11226 %}
11227
11228 // This pattern is automatically generated from aarch64_ad.m4
11229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11230 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11231 iRegL src1, iRegL src2,
11232 immI src3) %{
11233 match(Set dst (AddL src1 (LShiftL src2 src3)));
11234
11235 ins_cost(1.9 * INSN_COST);
11236 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11237
11238 ins_encode %{
11239 __ add(as_Register($dst$$reg),
11240 as_Register($src1$$reg),
11241 as_Register($src2$$reg),
11242 Assembler::LSL,
11243 $src3$$constant & 0x3f);
11244 %}
11245
11246 ins_pipe(ialu_reg_reg_shift);
11247 %}
11248
11249 // This pattern is automatically generated from aarch64_ad.m4
11250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11251 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11252 iRegIorL2I src1, iRegIorL2I src2,
11253 immI src3) %{
11254 match(Set dst (SubI src1 (URShiftI src2 src3)));
11255
11256 ins_cost(1.9 * INSN_COST);
11257 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11258
11259 ins_encode %{
11260 __ subw(as_Register($dst$$reg),
11261 as_Register($src1$$reg),
11262 as_Register($src2$$reg),
11263 Assembler::LSR,
11264 $src3$$constant & 0x1f);
11265 %}
11266
11267 ins_pipe(ialu_reg_reg_shift);
11268 %}
11269
11270 // This pattern is automatically generated from aarch64_ad.m4
11271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11272 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11273 iRegL src1, iRegL src2,
11274 immI src3) %{
11275 match(Set dst (SubL src1 (URShiftL src2 src3)));
11276
11277 ins_cost(1.9 * INSN_COST);
11278 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11279
11280 ins_encode %{
11281 __ sub(as_Register($dst$$reg),
11282 as_Register($src1$$reg),
11283 as_Register($src2$$reg),
11284 Assembler::LSR,
11285 $src3$$constant & 0x3f);
11286 %}
11287
11288 ins_pipe(ialu_reg_reg_shift);
11289 %}
11290
11291 // This pattern is automatically generated from aarch64_ad.m4
11292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11293 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11294 iRegIorL2I src1, iRegIorL2I src2,
11295 immI src3) %{
11296 match(Set dst (SubI src1 (RShiftI src2 src3)));
11297
11298 ins_cost(1.9 * INSN_COST);
11299 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11300
11301 ins_encode %{
11302 __ subw(as_Register($dst$$reg),
11303 as_Register($src1$$reg),
11304 as_Register($src2$$reg),
11305 Assembler::ASR,
11306 $src3$$constant & 0x1f);
11307 %}
11308
11309 ins_pipe(ialu_reg_reg_shift);
11310 %}
11311
11312 // This pattern is automatically generated from aarch64_ad.m4
11313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11314 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11315 iRegL src1, iRegL src2,
11316 immI src3) %{
11317 match(Set dst (SubL src1 (RShiftL src2 src3)));
11318
11319 ins_cost(1.9 * INSN_COST);
11320 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11321
11322 ins_encode %{
11323 __ sub(as_Register($dst$$reg),
11324 as_Register($src1$$reg),
11325 as_Register($src2$$reg),
11326 Assembler::ASR,
11327 $src3$$constant & 0x3f);
11328 %}
11329
11330 ins_pipe(ialu_reg_reg_shift);
11331 %}
11332
11333 // This pattern is automatically generated from aarch64_ad.m4
11334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11335 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11336 iRegIorL2I src1, iRegIorL2I src2,
11337 immI src3) %{
11338 match(Set dst (SubI src1 (LShiftI src2 src3)));
11339
11340 ins_cost(1.9 * INSN_COST);
11341 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11342
11343 ins_encode %{
11344 __ subw(as_Register($dst$$reg),
11345 as_Register($src1$$reg),
11346 as_Register($src2$$reg),
11347 Assembler::LSL,
11348 $src3$$constant & 0x1f);
11349 %}
11350
11351 ins_pipe(ialu_reg_reg_shift);
11352 %}
11353
11354 // This pattern is automatically generated from aarch64_ad.m4
11355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11356 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11357 iRegL src1, iRegL src2,
11358 immI src3) %{
11359 match(Set dst (SubL src1 (LShiftL src2 src3)));
11360
11361 ins_cost(1.9 * INSN_COST);
11362 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11363
11364 ins_encode %{
11365 __ sub(as_Register($dst$$reg),
11366 as_Register($src1$$reg),
11367 as_Register($src2$$reg),
11368 Assembler::LSL,
11369 $src3$$constant & 0x3f);
11370 %}
11371
11372 ins_pipe(ialu_reg_reg_shift);
11373 %}
11374
11375 // This pattern is automatically generated from aarch64_ad.m4
11376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11377
11378 // Shift Left followed by Shift Right.
11379 // This idiom is used by the compiler for the i2b bytecode etc.
11380 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11381 %{
11382 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11383 ins_cost(INSN_COST * 2);
11384 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11385 ins_encode %{
11386 int lshift = $lshift_count$$constant & 63;
11387 int rshift = $rshift_count$$constant & 63;
11388 int s = 63 - lshift;
11389 int r = (rshift - lshift) & 63;
11390 __ sbfm(as_Register($dst$$reg),
11391 as_Register($src$$reg),
11392 r, s);
11393 %}
11394
11395 ins_pipe(ialu_reg_shift);
11396 %}
11397
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400
11401 // Shift Left followed by Shift Right.
11402 // This idiom is used by the compiler for the i2b bytecode etc.
11403 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11404 %{
11405 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11406 ins_cost(INSN_COST * 2);
11407 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11408 ins_encode %{
11409 int lshift = $lshift_count$$constant & 31;
11410 int rshift = $rshift_count$$constant & 31;
11411 int s = 31 - lshift;
11412 int r = (rshift - lshift) & 31;
11413 __ sbfmw(as_Register($dst$$reg),
11414 as_Register($src$$reg),
11415 r, s);
11416 %}
11417
11418 ins_pipe(ialu_reg_shift);
11419 %}
11420
11421 // This pattern is automatically generated from aarch64_ad.m4
11422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11423
11424 // Shift Left followed by Shift Right.
11425 // This idiom is used by the compiler for the i2b bytecode etc.
11426 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11427 %{
11428 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11429 ins_cost(INSN_COST * 2);
11430 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11431 ins_encode %{
11432 int lshift = $lshift_count$$constant & 63;
11433 int rshift = $rshift_count$$constant & 63;
11434 int s = 63 - lshift;
11435 int r = (rshift - lshift) & 63;
11436 __ ubfm(as_Register($dst$$reg),
11437 as_Register($src$$reg),
11438 r, s);
11439 %}
11440
11441 ins_pipe(ialu_reg_shift);
11442 %}
11443
11444 // This pattern is automatically generated from aarch64_ad.m4
11445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11446
11447 // Shift Left followed by Shift Right.
11448 // This idiom is used by the compiler for the i2b bytecode etc.
11449 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11450 %{
11451 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11452 ins_cost(INSN_COST * 2);
11453 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11454 ins_encode %{
11455 int lshift = $lshift_count$$constant & 31;
11456 int rshift = $rshift_count$$constant & 31;
11457 int s = 31 - lshift;
11458 int r = (rshift - lshift) & 31;
11459 __ ubfmw(as_Register($dst$$reg),
11460 as_Register($src$$reg),
11461 r, s);
11462 %}
11463
11464 ins_pipe(ialu_reg_shift);
11465 %}
11466
11467 // Bitfield extract with shift & mask
11468
11469 // This pattern is automatically generated from aarch64_ad.m4
11470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11471 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11472 %{
11473 match(Set dst (AndI (URShiftI src rshift) mask));
11474 // Make sure we are not going to exceed what ubfxw can do.
11475 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11476
11477 ins_cost(INSN_COST);
11478 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11479 ins_encode %{
11480 int rshift = $rshift$$constant & 31;
11481 intptr_t mask = $mask$$constant;
11482 int width = exact_log2(mask+1);
11483 __ ubfxw(as_Register($dst$$reg),
11484 as_Register($src$$reg), rshift, width);
11485 %}
11486 ins_pipe(ialu_reg_shift);
11487 %}
11488
11489 // This pattern is automatically generated from aarch64_ad.m4
11490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11491 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11492 %{
11493 match(Set dst (AndL (URShiftL src rshift) mask));
11494 // Make sure we are not going to exceed what ubfx can do.
11495 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11496
11497 ins_cost(INSN_COST);
11498 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11499 ins_encode %{
11500 int rshift = $rshift$$constant & 63;
11501 intptr_t mask = $mask$$constant;
11502 int width = exact_log2_long(mask+1);
11503 __ ubfx(as_Register($dst$$reg),
11504 as_Register($src$$reg), rshift, width);
11505 %}
11506 ins_pipe(ialu_reg_shift);
11507 %}
11508
11509
11510 // This pattern is automatically generated from aarch64_ad.m4
11511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11512
11513 // We can use ubfx when extending an And with a mask when we know mask
11514 // is positive. We know that because immI_bitmask guarantees it.
11515 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11516 %{
11517 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11518 // Make sure we are not going to exceed what ubfxw can do.
11519 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11520
11521 ins_cost(INSN_COST * 2);
11522 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11523 ins_encode %{
11524 int rshift = $rshift$$constant & 31;
11525 intptr_t mask = $mask$$constant;
11526 int width = exact_log2(mask+1);
11527 __ ubfx(as_Register($dst$$reg),
11528 as_Register($src$$reg), rshift, width);
11529 %}
11530 ins_pipe(ialu_reg_shift);
11531 %}
11532
11533
11534 // This pattern is automatically generated from aarch64_ad.m4
11535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11536
11537 // We can use ubfiz when masking by a positive number and then left shifting the result.
11538 // We know that the mask is positive because immI_bitmask guarantees it.
11539 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11540 %{
11541 match(Set dst (LShiftI (AndI src mask) lshift));
11542 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11543
11544 ins_cost(INSN_COST);
11545 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11546 ins_encode %{
11547 int lshift = $lshift$$constant & 31;
11548 intptr_t mask = $mask$$constant;
11549 int width = exact_log2(mask+1);
11550 __ ubfizw(as_Register($dst$$reg),
11551 as_Register($src$$reg), lshift, width);
11552 %}
11553 ins_pipe(ialu_reg_shift);
11554 %}
11555
11556 // This pattern is automatically generated from aarch64_ad.m4
11557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11558
11559 // We can use ubfiz when masking by a positive number and then left shifting the result.
11560 // We know that the mask is positive because immL_bitmask guarantees it.
11561 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11562 %{
11563 match(Set dst (LShiftL (AndL src mask) lshift));
11564 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11565
11566 ins_cost(INSN_COST);
11567 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11568 ins_encode %{
11569 int lshift = $lshift$$constant & 63;
11570 intptr_t mask = $mask$$constant;
11571 int width = exact_log2_long(mask+1);
11572 __ ubfiz(as_Register($dst$$reg),
11573 as_Register($src$$reg), lshift, width);
11574 %}
11575 ins_pipe(ialu_reg_shift);
11576 %}
11577
11578 // This pattern is automatically generated from aarch64_ad.m4
11579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11580
11581 // We can use ubfiz when masking by a positive number and then left shifting the result.
11582 // We know that the mask is positive because immI_bitmask guarantees it.
11583 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11584 %{
11585 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11586 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11587
11588 ins_cost(INSN_COST);
11589 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11590 ins_encode %{
11591 int lshift = $lshift$$constant & 31;
11592 intptr_t mask = $mask$$constant;
11593 int width = exact_log2(mask+1);
11594 __ ubfizw(as_Register($dst$$reg),
11595 as_Register($src$$reg), lshift, width);
11596 %}
11597 ins_pipe(ialu_reg_shift);
11598 %}
11599
11600 // This pattern is automatically generated from aarch64_ad.m4
11601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11602
11603 // We can use ubfiz when masking by a positive number and then left shifting the result.
11604 // We know that the mask is positive because immL_bitmask guarantees it.
11605 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11606 %{
11607 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11608 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11609
11610 ins_cost(INSN_COST);
11611 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11612 ins_encode %{
11613 int lshift = $lshift$$constant & 63;
11614 intptr_t mask = $mask$$constant;
11615 int width = exact_log2_long(mask+1);
11616 __ ubfiz(as_Register($dst$$reg),
11617 as_Register($src$$reg), lshift, width);
11618 %}
11619 ins_pipe(ialu_reg_shift);
11620 %}
11621
11622
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625
11626 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11627 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11628 %{
11629 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11630 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11631
11632 ins_cost(INSN_COST);
11633 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11634 ins_encode %{
11635 int lshift = $lshift$$constant & 63;
11636 intptr_t mask = $mask$$constant;
11637 int width = exact_log2(mask+1);
11638 __ ubfiz(as_Register($dst$$reg),
11639 as_Register($src$$reg), lshift, width);
11640 %}
11641 ins_pipe(ialu_reg_shift);
11642 %}
11643
11644 // This pattern is automatically generated from aarch64_ad.m4
11645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11646
11647 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11648 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11649 %{
11650 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11651 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11652
11653 ins_cost(INSN_COST);
11654 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11655 ins_encode %{
11656 int lshift = $lshift$$constant & 31;
11657 intptr_t mask = $mask$$constant;
11658 int width = exact_log2(mask+1);
11659 __ ubfiz(as_Register($dst$$reg),
11660 as_Register($src$$reg), lshift, width);
11661 %}
11662 ins_pipe(ialu_reg_shift);
11663 %}
11664
11665 // This pattern is automatically generated from aarch64_ad.m4
11666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11667
11668 // Can skip int2long conversions after AND with small bitmask
11669 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11670 %{
11671 match(Set dst (ConvI2L (AndI src msk)));
11672 ins_cost(INSN_COST);
11673 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11674 ins_encode %{
11675 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11676 %}
11677 ins_pipe(ialu_reg_shift);
11678 %}
11679
11680
11681 // Rotations
11682
11683 // This pattern is automatically generated from aarch64_ad.m4
11684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11685 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11686 %{
11687 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11688 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11689
11690 ins_cost(INSN_COST);
11691 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11692
11693 ins_encode %{
11694 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11695 $rshift$$constant & 63);
11696 %}
11697 ins_pipe(ialu_reg_reg_extr);
11698 %}
11699
11700
11701 // This pattern is automatically generated from aarch64_ad.m4
11702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11703 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11704 %{
11705 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11706 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11707
11708 ins_cost(INSN_COST);
11709 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11710
11711 ins_encode %{
11712 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11713 $rshift$$constant & 31);
11714 %}
11715 ins_pipe(ialu_reg_reg_extr);
11716 %}
11717
11718
11719 // This pattern is automatically generated from aarch64_ad.m4
11720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11721 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11722 %{
11723 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11724 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11725
11726 ins_cost(INSN_COST);
11727 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11728
11729 ins_encode %{
11730 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11731 $rshift$$constant & 63);
11732 %}
11733 ins_pipe(ialu_reg_reg_extr);
11734 %}
11735
11736
11737 // This pattern is automatically generated from aarch64_ad.m4
11738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11739 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11740 %{
11741 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11742 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11743
11744 ins_cost(INSN_COST);
11745 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11746
11747 ins_encode %{
11748 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11749 $rshift$$constant & 31);
11750 %}
11751 ins_pipe(ialu_reg_reg_extr);
11752 %}
11753
11754 // This pattern is automatically generated from aarch64_ad.m4
11755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11756 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11757 %{
11758 match(Set dst (RotateRight src shift));
11759
11760 ins_cost(INSN_COST);
11761 format %{ "ror $dst, $src, $shift" %}
11762
11763 ins_encode %{
11764 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11765 $shift$$constant & 0x1f);
11766 %}
11767 ins_pipe(ialu_reg_reg_vshift);
11768 %}
11769
11770 // This pattern is automatically generated from aarch64_ad.m4
11771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11772 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11773 %{
11774 match(Set dst (RotateRight src shift));
11775
11776 ins_cost(INSN_COST);
11777 format %{ "ror $dst, $src, $shift" %}
11778
11779 ins_encode %{
11780 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11781 $shift$$constant & 0x3f);
11782 %}
11783 ins_pipe(ialu_reg_reg_vshift);
11784 %}
11785
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11789 %{
11790 match(Set dst (RotateRight src shift));
11791
11792 ins_cost(INSN_COST);
11793 format %{ "ror $dst, $src, $shift" %}
11794
11795 ins_encode %{
11796 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11797 %}
11798 ins_pipe(ialu_reg_reg_vshift);
11799 %}
11800
11801 // This pattern is automatically generated from aarch64_ad.m4
11802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11803 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11804 %{
11805 match(Set dst (RotateRight src shift));
11806
11807 ins_cost(INSN_COST);
11808 format %{ "ror $dst, $src, $shift" %}
11809
11810 ins_encode %{
11811 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11812 %}
11813 ins_pipe(ialu_reg_reg_vshift);
11814 %}
11815
11816 // This pattern is automatically generated from aarch64_ad.m4
11817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11818 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11819 %{
11820 match(Set dst (RotateLeft src shift));
11821
11822 ins_cost(INSN_COST);
11823 format %{ "rol $dst, $src, $shift" %}
11824
11825 ins_encode %{
11826 __ subw(rscratch1, zr, as_Register($shift$$reg));
11827 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11828 %}
11829 ins_pipe(ialu_reg_reg_vshift);
11830 %}
11831
11832 // This pattern is automatically generated from aarch64_ad.m4
11833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11834 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11835 %{
11836 match(Set dst (RotateLeft src shift));
11837
11838 ins_cost(INSN_COST);
11839 format %{ "rol $dst, $src, $shift" %}
11840
11841 ins_encode %{
11842 __ subw(rscratch1, zr, as_Register($shift$$reg));
11843 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11844 %}
11845 ins_pipe(ialu_reg_reg_vshift);
11846 %}
11847
11848
11849 // Add/subtract (extended)
11850
11851 // This pattern is automatically generated from aarch64_ad.m4
11852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11853 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11854 %{
11855 match(Set dst (AddL src1 (ConvI2L src2)));
11856 ins_cost(INSN_COST);
11857 format %{ "add $dst, $src1, $src2, sxtw" %}
11858
11859 ins_encode %{
11860 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11861 as_Register($src2$$reg), ext::sxtw);
11862 %}
11863 ins_pipe(ialu_reg_reg);
11864 %}
11865
11866 // This pattern is automatically generated from aarch64_ad.m4
11867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11868 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11869 %{
11870 match(Set dst (SubL src1 (ConvI2L src2)));
11871 ins_cost(INSN_COST);
11872 format %{ "sub $dst, $src1, $src2, sxtw" %}
11873
11874 ins_encode %{
11875 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11876 as_Register($src2$$reg), ext::sxtw);
11877 %}
11878 ins_pipe(ialu_reg_reg);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11884 %{
11885 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11886 ins_cost(INSN_COST);
11887 format %{ "add $dst, $src1, $src2, sxth" %}
11888
11889 ins_encode %{
11890 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11891 as_Register($src2$$reg), ext::sxth);
11892 %}
11893 ins_pipe(ialu_reg_reg);
11894 %}
11895
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11899 %{
11900 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11901 ins_cost(INSN_COST);
11902 format %{ "add $dst, $src1, $src2, sxtb" %}
11903
11904 ins_encode %{
11905 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11906 as_Register($src2$$reg), ext::sxtb);
11907 %}
11908 ins_pipe(ialu_reg_reg);
11909 %}
11910
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11914 %{
11915 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11916 ins_cost(INSN_COST);
11917 format %{ "add $dst, $src1, $src2, uxtb" %}
11918
11919 ins_encode %{
11920 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11921 as_Register($src2$$reg), ext::uxtb);
11922 %}
11923 ins_pipe(ialu_reg_reg);
11924 %}
11925
11926 // This pattern is automatically generated from aarch64_ad.m4
11927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11928 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11929 %{
11930 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11931 ins_cost(INSN_COST);
11932 format %{ "add $dst, $src1, $src2, sxth" %}
11933
11934 ins_encode %{
11935 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11936 as_Register($src2$$reg), ext::sxth);
11937 %}
11938 ins_pipe(ialu_reg_reg);
11939 %}
11940
11941 // This pattern is automatically generated from aarch64_ad.m4
11942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11943 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11944 %{
11945 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11946 ins_cost(INSN_COST);
11947 format %{ "add $dst, $src1, $src2, sxtw" %}
11948
11949 ins_encode %{
11950 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11951 as_Register($src2$$reg), ext::sxtw);
11952 %}
11953 ins_pipe(ialu_reg_reg);
11954 %}
11955
11956 // This pattern is automatically generated from aarch64_ad.m4
11957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11958 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11959 %{
11960 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11961 ins_cost(INSN_COST);
11962 format %{ "add $dst, $src1, $src2, sxtb" %}
11963
11964 ins_encode %{
11965 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11966 as_Register($src2$$reg), ext::sxtb);
11967 %}
11968 ins_pipe(ialu_reg_reg);
11969 %}
11970
11971 // This pattern is automatically generated from aarch64_ad.m4
11972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11973 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11974 %{
11975 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11976 ins_cost(INSN_COST);
11977 format %{ "add $dst, $src1, $src2, uxtb" %}
11978
11979 ins_encode %{
11980 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11981 as_Register($src2$$reg), ext::uxtb);
11982 %}
11983 ins_pipe(ialu_reg_reg);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11989 %{
11990 match(Set dst (AddI src1 (AndI src2 mask)));
11991 ins_cost(INSN_COST);
11992 format %{ "addw $dst, $src1, $src2, uxtb" %}
11993
11994 ins_encode %{
11995 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11996 as_Register($src2$$reg), ext::uxtb);
11997 %}
11998 ins_pipe(ialu_reg_reg);
11999 %}
12000
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12004 %{
12005 match(Set dst (AddI src1 (AndI src2 mask)));
12006 ins_cost(INSN_COST);
12007 format %{ "addw $dst, $src1, $src2, uxth" %}
12008
12009 ins_encode %{
12010 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12011 as_Register($src2$$reg), ext::uxth);
12012 %}
12013 ins_pipe(ialu_reg_reg);
12014 %}
12015
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12019 %{
12020 match(Set dst (AddL src1 (AndL src2 mask)));
12021 ins_cost(INSN_COST);
12022 format %{ "add $dst, $src1, $src2, uxtb" %}
12023
12024 ins_encode %{
12025 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12026 as_Register($src2$$reg), ext::uxtb);
12027 %}
12028 ins_pipe(ialu_reg_reg);
12029 %}
12030
12031 // This pattern is automatically generated from aarch64_ad.m4
12032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12033 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12034 %{
12035 match(Set dst (AddL src1 (AndL src2 mask)));
12036 ins_cost(INSN_COST);
12037 format %{ "add $dst, $src1, $src2, uxth" %}
12038
12039 ins_encode %{
12040 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12041 as_Register($src2$$reg), ext::uxth);
12042 %}
12043 ins_pipe(ialu_reg_reg);
12044 %}
12045
12046 // This pattern is automatically generated from aarch64_ad.m4
12047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12048 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12049 %{
12050 match(Set dst (AddL src1 (AndL src2 mask)));
12051 ins_cost(INSN_COST);
12052 format %{ "add $dst, $src1, $src2, uxtw" %}
12053
12054 ins_encode %{
12055 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12056 as_Register($src2$$reg), ext::uxtw);
12057 %}
12058 ins_pipe(ialu_reg_reg);
12059 %}
12060
12061 // This pattern is automatically generated from aarch64_ad.m4
12062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12063 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12064 %{
12065 match(Set dst (SubI src1 (AndI src2 mask)));
12066 ins_cost(INSN_COST);
12067 format %{ "subw $dst, $src1, $src2, uxtb" %}
12068
12069 ins_encode %{
12070 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12071 as_Register($src2$$reg), ext::uxtb);
12072 %}
12073 ins_pipe(ialu_reg_reg);
12074 %}
12075
12076 // This pattern is automatically generated from aarch64_ad.m4
12077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12078 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12079 %{
12080 match(Set dst (SubI src1 (AndI src2 mask)));
12081 ins_cost(INSN_COST);
12082 format %{ "subw $dst, $src1, $src2, uxth" %}
12083
12084 ins_encode %{
12085 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12086 as_Register($src2$$reg), ext::uxth);
12087 %}
12088 ins_pipe(ialu_reg_reg);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12094 %{
12095 match(Set dst (SubL src1 (AndL src2 mask)));
12096 ins_cost(INSN_COST);
12097 format %{ "sub $dst, $src1, $src2, uxtb" %}
12098
12099 ins_encode %{
12100 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12101 as_Register($src2$$reg), ext::uxtb);
12102 %}
12103 ins_pipe(ialu_reg_reg);
12104 %}
12105
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12109 %{
12110 match(Set dst (SubL src1 (AndL src2 mask)));
12111 ins_cost(INSN_COST);
12112 format %{ "sub $dst, $src1, $src2, uxth" %}
12113
12114 ins_encode %{
12115 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12116 as_Register($src2$$reg), ext::uxth);
12117 %}
12118 ins_pipe(ialu_reg_reg);
12119 %}
12120
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12124 %{
12125 match(Set dst (SubL src1 (AndL src2 mask)));
12126 ins_cost(INSN_COST);
12127 format %{ "sub $dst, $src1, $src2, uxtw" %}
12128
12129 ins_encode %{
12130 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12131 as_Register($src2$$reg), ext::uxtw);
12132 %}
12133 ins_pipe(ialu_reg_reg);
12134 %}
12135
12136
12137 // This pattern is automatically generated from aarch64_ad.m4
12138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12139 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12140 %{
12141 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12142 ins_cost(1.9 * INSN_COST);
12143 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12144
12145 ins_encode %{
12146 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12147 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12148 %}
12149 ins_pipe(ialu_reg_reg_shift);
12150 %}
12151
12152 // This pattern is automatically generated from aarch64_ad.m4
12153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12154 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12155 %{
12156 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12157 ins_cost(1.9 * INSN_COST);
12158 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12159
12160 ins_encode %{
12161 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12162 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12163 %}
12164 ins_pipe(ialu_reg_reg_shift);
12165 %}
12166
12167 // This pattern is automatically generated from aarch64_ad.m4
12168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12169 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12170 %{
12171 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12172 ins_cost(1.9 * INSN_COST);
12173 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12174
12175 ins_encode %{
12176 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12177 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12178 %}
12179 ins_pipe(ialu_reg_reg_shift);
12180 %}
12181
12182 // This pattern is automatically generated from aarch64_ad.m4
12183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12184 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12185 %{
12186 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12187 ins_cost(1.9 * INSN_COST);
12188 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12189
12190 ins_encode %{
12191 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12192 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12193 %}
12194 ins_pipe(ialu_reg_reg_shift);
12195 %}
12196
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12200 %{
12201 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12202 ins_cost(1.9 * INSN_COST);
12203 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12204
12205 ins_encode %{
12206 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12207 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12208 %}
12209 ins_pipe(ialu_reg_reg_shift);
12210 %}
12211
12212 // This pattern is automatically generated from aarch64_ad.m4
12213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12214 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12215 %{
12216 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12217 ins_cost(1.9 * INSN_COST);
12218 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12219
12220 ins_encode %{
12221 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12222 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12223 %}
12224 ins_pipe(ialu_reg_reg_shift);
12225 %}
12226
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12230 %{
12231 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12232 ins_cost(1.9 * INSN_COST);
12233 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12234
12235 ins_encode %{
12236 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12237 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12238 %}
12239 ins_pipe(ialu_reg_reg_shift);
12240 %}
12241
12242 // This pattern is automatically generated from aarch64_ad.m4
12243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12244 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12245 %{
12246 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12247 ins_cost(1.9 * INSN_COST);
12248 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12249
12250 ins_encode %{
12251 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12252 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12253 %}
12254 ins_pipe(ialu_reg_reg_shift);
12255 %}
12256
12257 // This pattern is automatically generated from aarch64_ad.m4
12258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12259 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12260 %{
12261 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12262 ins_cost(1.9 * INSN_COST);
12263 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12264
12265 ins_encode %{
12266 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12267 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12268 %}
12269 ins_pipe(ialu_reg_reg_shift);
12270 %}
12271
12272 // This pattern is automatically generated from aarch64_ad.m4
12273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12274 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12275 %{
12276 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12277 ins_cost(1.9 * INSN_COST);
12278 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12279
12280 ins_encode %{
12281 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12282 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12283 %}
12284 ins_pipe(ialu_reg_reg_shift);
12285 %}
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12290 %{
12291 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12292 ins_cost(1.9 * INSN_COST);
12293 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12294
12295 ins_encode %{
12296 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12297 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12298 %}
12299 ins_pipe(ialu_reg_reg_shift);
12300 %}
12301
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12305 %{
12306 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12307 ins_cost(1.9 * INSN_COST);
12308 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12309
12310 ins_encode %{
12311 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12312 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12313 %}
12314 ins_pipe(ialu_reg_reg_shift);
12315 %}
12316
12317 // This pattern is automatically generated from aarch64_ad.m4
12318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12319 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12320 %{
12321 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12322 ins_cost(1.9 * INSN_COST);
12323 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12324
12325 ins_encode %{
12326 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12327 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12328 %}
12329 ins_pipe(ialu_reg_reg_shift);
12330 %}
12331
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12335 %{
12336 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12337 ins_cost(1.9 * INSN_COST);
12338 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12339
12340 ins_encode %{
12341 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12342 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12343 %}
12344 ins_pipe(ialu_reg_reg_shift);
12345 %}
12346
12347 // This pattern is automatically generated from aarch64_ad.m4
12348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12349 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12350 %{
12351 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12352 ins_cost(1.9 * INSN_COST);
12353 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12354
12355 ins_encode %{
12356 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12357 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12358 %}
12359 ins_pipe(ialu_reg_reg_shift);
12360 %}
12361
12362 // This pattern is automatically generated from aarch64_ad.m4
12363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12364 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12365 %{
12366 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12367 ins_cost(1.9 * INSN_COST);
12368 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12369
12370 ins_encode %{
12371 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12372 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12373 %}
12374 ins_pipe(ialu_reg_reg_shift);
12375 %}
12376
12377 // This pattern is automatically generated from aarch64_ad.m4
12378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12379 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12380 %{
12381 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12382 ins_cost(1.9 * INSN_COST);
12383 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12384
12385 ins_encode %{
12386 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12387 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12388 %}
12389 ins_pipe(ialu_reg_reg_shift);
12390 %}
12391
12392 // This pattern is automatically generated from aarch64_ad.m4
12393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12394 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12395 %{
12396 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12397 ins_cost(1.9 * INSN_COST);
12398 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12399
12400 ins_encode %{
12401 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12402 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12403 %}
12404 ins_pipe(ialu_reg_reg_shift);
12405 %}
12406
12407 // This pattern is automatically generated from aarch64_ad.m4
12408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12409 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12410 %{
12411 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12412 ins_cost(1.9 * INSN_COST);
12413 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12414
12415 ins_encode %{
12416 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12417 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12418 %}
12419 ins_pipe(ialu_reg_reg_shift);
12420 %}
12421
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12425 %{
12426 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12427 ins_cost(1.9 * INSN_COST);
12428 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12429
12430 ins_encode %{
12431 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12432 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12433 %}
12434 ins_pipe(ialu_reg_reg_shift);
12435 %}
12436
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12440 %{
12441 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12442 ins_cost(1.9 * INSN_COST);
12443 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12444
12445 ins_encode %{
12446 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12447 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12448 %}
12449 ins_pipe(ialu_reg_reg_shift);
12450 %}
12451
12452 // This pattern is automatically generated from aarch64_ad.m4
12453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12454 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12455 %{
12456 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12457 ins_cost(1.9 * INSN_COST);
12458 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12459
12460 ins_encode %{
12461 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12462 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12463 %}
12464 ins_pipe(ialu_reg_reg_shift);
12465 %}
12466
12467 // This pattern is automatically generated from aarch64_ad.m4
12468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12469 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12470 %{
12471 effect(DEF dst, USE src1, USE src2, USE cr);
12472 ins_cost(INSN_COST * 2);
12473 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12474
12475 ins_encode %{
12476 __ cselw($dst$$Register,
12477 $src1$$Register,
12478 $src2$$Register,
12479 Assembler::LT);
12480 %}
12481 ins_pipe(icond_reg_reg);
12482 %}
12483
12484 // This pattern is automatically generated from aarch64_ad.m4
12485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12486 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12487 %{
12488 effect(DEF dst, USE src1, USE src2, USE cr);
12489 ins_cost(INSN_COST * 2);
12490 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12491
12492 ins_encode %{
12493 __ cselw($dst$$Register,
12494 $src1$$Register,
12495 $src2$$Register,
12496 Assembler::GT);
12497 %}
12498 ins_pipe(icond_reg_reg);
12499 %}
12500
12501 // This pattern is automatically generated from aarch64_ad.m4
12502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12503 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12504 %{
12505 effect(DEF dst, USE src1, USE cr);
12506 ins_cost(INSN_COST * 2);
12507 format %{ "cselw $dst, $src1, zr lt\t" %}
12508
12509 ins_encode %{
12510 __ cselw($dst$$Register,
12511 $src1$$Register,
12512 zr,
12513 Assembler::LT);
12514 %}
12515 ins_pipe(icond_reg);
12516 %}
12517
12518 // This pattern is automatically generated from aarch64_ad.m4
12519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12520 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12521 %{
12522 effect(DEF dst, USE src1, USE cr);
12523 ins_cost(INSN_COST * 2);
12524 format %{ "cselw $dst, $src1, zr gt\t" %}
12525
12526 ins_encode %{
12527 __ cselw($dst$$Register,
12528 $src1$$Register,
12529 zr,
12530 Assembler::GT);
12531 %}
12532 ins_pipe(icond_reg);
12533 %}
12534
12535 // This pattern is automatically generated from aarch64_ad.m4
12536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12537 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12538 %{
12539 effect(DEF dst, USE src1, USE cr);
12540 ins_cost(INSN_COST * 2);
12541 format %{ "csincw $dst, $src1, zr le\t" %}
12542
12543 ins_encode %{
12544 __ csincw($dst$$Register,
12545 $src1$$Register,
12546 zr,
12547 Assembler::LE);
12548 %}
12549 ins_pipe(icond_reg);
12550 %}
12551
12552 // This pattern is automatically generated from aarch64_ad.m4
12553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12554 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12555 %{
12556 effect(DEF dst, USE src1, USE cr);
12557 ins_cost(INSN_COST * 2);
12558 format %{ "csincw $dst, $src1, zr gt\t" %}
12559
12560 ins_encode %{
12561 __ csincw($dst$$Register,
12562 $src1$$Register,
12563 zr,
12564 Assembler::GT);
12565 %}
12566 ins_pipe(icond_reg);
12567 %}
12568
12569 // This pattern is automatically generated from aarch64_ad.m4
12570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12571 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12572 %{
12573 effect(DEF dst, USE src1, USE cr);
12574 ins_cost(INSN_COST * 2);
12575 format %{ "csinvw $dst, $src1, zr lt\t" %}
12576
12577 ins_encode %{
12578 __ csinvw($dst$$Register,
12579 $src1$$Register,
12580 zr,
12581 Assembler::LT);
12582 %}
12583 ins_pipe(icond_reg);
12584 %}
12585
12586 // This pattern is automatically generated from aarch64_ad.m4
12587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12588 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12589 %{
12590 effect(DEF dst, USE src1, USE cr);
12591 ins_cost(INSN_COST * 2);
12592 format %{ "csinvw $dst, $src1, zr ge\t" %}
12593
12594 ins_encode %{
12595 __ csinvw($dst$$Register,
12596 $src1$$Register,
12597 zr,
12598 Assembler::GE);
12599 %}
12600 ins_pipe(icond_reg);
12601 %}
12602
12603 // This pattern is automatically generated from aarch64_ad.m4
12604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12605 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12606 %{
12607 match(Set dst (MinI src imm));
12608 ins_cost(INSN_COST * 3);
12609 expand %{
12610 rFlagsReg cr;
12611 compI_reg_imm0(cr, src);
12612 cmovI_reg_imm0_lt(dst, src, cr);
12613 %}
12614 %}
12615
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12619 %{
12620 match(Set dst (MinI imm src));
12621 ins_cost(INSN_COST * 3);
12622 expand %{
12623 rFlagsReg cr;
12624 compI_reg_imm0(cr, src);
12625 cmovI_reg_imm0_lt(dst, src, cr);
12626 %}
12627 %}
12628
12629 // This pattern is automatically generated from aarch64_ad.m4
12630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12631 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12632 %{
12633 match(Set dst (MinI src imm));
12634 ins_cost(INSN_COST * 3);
12635 expand %{
12636 rFlagsReg cr;
12637 compI_reg_imm0(cr, src);
12638 cmovI_reg_imm1_le(dst, src, cr);
12639 %}
12640 %}
12641
12642 // This pattern is automatically generated from aarch64_ad.m4
12643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12644 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12645 %{
12646 match(Set dst (MinI imm src));
12647 ins_cost(INSN_COST * 3);
12648 expand %{
12649 rFlagsReg cr;
12650 compI_reg_imm0(cr, src);
12651 cmovI_reg_imm1_le(dst, src, cr);
12652 %}
12653 %}
12654
12655 // This pattern is automatically generated from aarch64_ad.m4
12656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12657 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12658 %{
12659 match(Set dst (MinI src imm));
12660 ins_cost(INSN_COST * 3);
12661 expand %{
12662 rFlagsReg cr;
12663 compI_reg_imm0(cr, src);
12664 cmovI_reg_immM1_lt(dst, src, cr);
12665 %}
12666 %}
12667
12668 // This pattern is automatically generated from aarch64_ad.m4
12669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12670 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12671 %{
12672 match(Set dst (MinI imm src));
12673 ins_cost(INSN_COST * 3);
12674 expand %{
12675 rFlagsReg cr;
12676 compI_reg_imm0(cr, src);
12677 cmovI_reg_immM1_lt(dst, src, cr);
12678 %}
12679 %}
12680
12681 // This pattern is automatically generated from aarch64_ad.m4
12682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12683 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12684 %{
12685 match(Set dst (MaxI src imm));
12686 ins_cost(INSN_COST * 3);
12687 expand %{
12688 rFlagsReg cr;
12689 compI_reg_imm0(cr, src);
12690 cmovI_reg_imm0_gt(dst, src, cr);
12691 %}
12692 %}
12693
12694 // This pattern is automatically generated from aarch64_ad.m4
12695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12696 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12697 %{
12698 match(Set dst (MaxI imm src));
12699 ins_cost(INSN_COST * 3);
12700 expand %{
12701 rFlagsReg cr;
12702 compI_reg_imm0(cr, src);
12703 cmovI_reg_imm0_gt(dst, src, cr);
12704 %}
12705 %}
12706
12707 // This pattern is automatically generated from aarch64_ad.m4
12708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12709 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12710 %{
12711 match(Set dst (MaxI src imm));
12712 ins_cost(INSN_COST * 3);
12713 expand %{
12714 rFlagsReg cr;
12715 compI_reg_imm0(cr, src);
12716 cmovI_reg_imm1_gt(dst, src, cr);
12717 %}
12718 %}
12719
12720 // This pattern is automatically generated from aarch64_ad.m4
12721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12722 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12723 %{
12724 match(Set dst (MaxI imm src));
12725 ins_cost(INSN_COST * 3);
12726 expand %{
12727 rFlagsReg cr;
12728 compI_reg_imm0(cr, src);
12729 cmovI_reg_imm1_gt(dst, src, cr);
12730 %}
12731 %}
12732
12733 // This pattern is automatically generated from aarch64_ad.m4
12734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12735 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12736 %{
12737 match(Set dst (MaxI src imm));
12738 ins_cost(INSN_COST * 3);
12739 expand %{
12740 rFlagsReg cr;
12741 compI_reg_imm0(cr, src);
12742 cmovI_reg_immM1_ge(dst, src, cr);
12743 %}
12744 %}
12745
12746 // This pattern is automatically generated from aarch64_ad.m4
12747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12748 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12749 %{
12750 match(Set dst (MaxI imm src));
12751 ins_cost(INSN_COST * 3);
12752 expand %{
12753 rFlagsReg cr;
12754 compI_reg_imm0(cr, src);
12755 cmovI_reg_immM1_ge(dst, src, cr);
12756 %}
12757 %}
12758
12759 // This pattern is automatically generated from aarch64_ad.m4
12760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12761 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12762 %{
12763 match(Set dst (ReverseI src));
12764 ins_cost(INSN_COST);
12765 format %{ "rbitw $dst, $src" %}
12766 ins_encode %{
12767 __ rbitw($dst$$Register, $src$$Register);
12768 %}
12769 ins_pipe(ialu_reg);
12770 %}
12771
12772 // This pattern is automatically generated from aarch64_ad.m4
12773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12774 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12775 %{
12776 match(Set dst (ReverseL src));
12777 ins_cost(INSN_COST);
12778 format %{ "rbit $dst, $src" %}
12779 ins_encode %{
12780 __ rbit($dst$$Register, $src$$Register);
12781 %}
12782 ins_pipe(ialu_reg);
12783 %}
12784
12785
12786 // END This section of the file is automatically generated. Do not edit --------------
12787
12788
12789 // ============================================================================
12790 // Floating Point Arithmetic Instructions
12791
12792 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12793 match(Set dst (AddHF src1 src2));
12794 format %{ "faddh $dst, $src1, $src2" %}
12795 ins_encode %{
12796 __ faddh($dst$$FloatRegister,
12797 $src1$$FloatRegister,
12798 $src2$$FloatRegister);
12799 %}
12800 ins_pipe(fp_dop_reg_reg_s);
12801 %}
12802
12803 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12804 match(Set dst (AddF src1 src2));
12805
12806 ins_cost(INSN_COST * 5);
12807 format %{ "fadds $dst, $src1, $src2" %}
12808
12809 ins_encode %{
12810 __ fadds(as_FloatRegister($dst$$reg),
12811 as_FloatRegister($src1$$reg),
12812 as_FloatRegister($src2$$reg));
12813 %}
12814
12815 ins_pipe(fp_dop_reg_reg_s);
12816 %}
12817
12818 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12819 match(Set dst (AddD src1 src2));
12820
12821 ins_cost(INSN_COST * 5);
12822 format %{ "faddd $dst, $src1, $src2" %}
12823
12824 ins_encode %{
12825 __ faddd(as_FloatRegister($dst$$reg),
12826 as_FloatRegister($src1$$reg),
12827 as_FloatRegister($src2$$reg));
12828 %}
12829
12830 ins_pipe(fp_dop_reg_reg_d);
12831 %}
12832
12833 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12834 match(Set dst (SubHF src1 src2));
12835 format %{ "fsubh $dst, $src1, $src2" %}
12836 ins_encode %{
12837 __ fsubh($dst$$FloatRegister,
12838 $src1$$FloatRegister,
12839 $src2$$FloatRegister);
12840 %}
12841 ins_pipe(fp_dop_reg_reg_s);
12842 %}
12843
12844 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12845 match(Set dst (SubF src1 src2));
12846
12847 ins_cost(INSN_COST * 5);
12848 format %{ "fsubs $dst, $src1, $src2" %}
12849
12850 ins_encode %{
12851 __ fsubs(as_FloatRegister($dst$$reg),
12852 as_FloatRegister($src1$$reg),
12853 as_FloatRegister($src2$$reg));
12854 %}
12855
12856 ins_pipe(fp_dop_reg_reg_s);
12857 %}
12858
12859 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12860 match(Set dst (SubD src1 src2));
12861
12862 ins_cost(INSN_COST * 5);
12863 format %{ "fsubd $dst, $src1, $src2" %}
12864
12865 ins_encode %{
12866 __ fsubd(as_FloatRegister($dst$$reg),
12867 as_FloatRegister($src1$$reg),
12868 as_FloatRegister($src2$$reg));
12869 %}
12870
12871 ins_pipe(fp_dop_reg_reg_d);
12872 %}
12873
12874 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12875 match(Set dst (MulHF src1 src2));
12876 format %{ "fmulh $dst, $src1, $src2" %}
12877 ins_encode %{
12878 __ fmulh($dst$$FloatRegister,
12879 $src1$$FloatRegister,
12880 $src2$$FloatRegister);
12881 %}
12882 ins_pipe(fp_dop_reg_reg_s);
12883 %}
12884
12885 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12886 match(Set dst (MulF src1 src2));
12887
12888 ins_cost(INSN_COST * 6);
12889 format %{ "fmuls $dst, $src1, $src2" %}
12890
12891 ins_encode %{
12892 __ fmuls(as_FloatRegister($dst$$reg),
12893 as_FloatRegister($src1$$reg),
12894 as_FloatRegister($src2$$reg));
12895 %}
12896
12897 ins_pipe(fp_dop_reg_reg_s);
12898 %}
12899
12900 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12901 match(Set dst (MulD src1 src2));
12902
12903 ins_cost(INSN_COST * 6);
12904 format %{ "fmuld $dst, $src1, $src2" %}
12905
12906 ins_encode %{
12907 __ fmuld(as_FloatRegister($dst$$reg),
12908 as_FloatRegister($src1$$reg),
12909 as_FloatRegister($src2$$reg));
12910 %}
12911
12912 ins_pipe(fp_dop_reg_reg_d);
12913 %}
12914
12915 // src1 * src2 + src3 (half-precision float)
12916 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12917 match(Set dst (FmaHF src3 (Binary src1 src2)));
12918 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12919 ins_encode %{
12920 assert(UseFMA, "Needs FMA instructions support.");
12921 __ fmaddh($dst$$FloatRegister,
12922 $src1$$FloatRegister,
12923 $src2$$FloatRegister,
12924 $src3$$FloatRegister);
12925 %}
12926 ins_pipe(pipe_class_default);
12927 %}
12928
12929 // src1 * src2 + src3
12930 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12931 match(Set dst (FmaF src3 (Binary src1 src2)));
12932
12933 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12934
12935 ins_encode %{
12936 assert(UseFMA, "Needs FMA instructions support.");
12937 __ fmadds(as_FloatRegister($dst$$reg),
12938 as_FloatRegister($src1$$reg),
12939 as_FloatRegister($src2$$reg),
12940 as_FloatRegister($src3$$reg));
12941 %}
12942
12943 ins_pipe(pipe_class_default);
12944 %}
12945
12946 // src1 * src2 + src3
12947 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12948 match(Set dst (FmaD src3 (Binary src1 src2)));
12949
12950 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12951
12952 ins_encode %{
12953 assert(UseFMA, "Needs FMA instructions support.");
12954 __ fmaddd(as_FloatRegister($dst$$reg),
12955 as_FloatRegister($src1$$reg),
12956 as_FloatRegister($src2$$reg),
12957 as_FloatRegister($src3$$reg));
12958 %}
12959
12960 ins_pipe(pipe_class_default);
12961 %}
12962
12963 // src1 * (-src2) + src3
12964 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12965 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12966 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12967
12968 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12969
12970 ins_encode %{
12971 assert(UseFMA, "Needs FMA instructions support.");
12972 __ fmsubs(as_FloatRegister($dst$$reg),
12973 as_FloatRegister($src1$$reg),
12974 as_FloatRegister($src2$$reg),
12975 as_FloatRegister($src3$$reg));
12976 %}
12977
12978 ins_pipe(pipe_class_default);
12979 %}
12980
12981 // src1 * (-src2) + src3
12982 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12983 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12984 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12985
12986 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12987
12988 ins_encode %{
12989 assert(UseFMA, "Needs FMA instructions support.");
12990 __ fmsubd(as_FloatRegister($dst$$reg),
12991 as_FloatRegister($src1$$reg),
12992 as_FloatRegister($src2$$reg),
12993 as_FloatRegister($src3$$reg));
12994 %}
12995
12996 ins_pipe(pipe_class_default);
12997 %}
12998
12999 // src1 * (-src2) - src3
13000 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13001 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13002 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13003
13004 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13005
13006 ins_encode %{
13007 assert(UseFMA, "Needs FMA instructions support.");
13008 __ fnmadds(as_FloatRegister($dst$$reg),
13009 as_FloatRegister($src1$$reg),
13010 as_FloatRegister($src2$$reg),
13011 as_FloatRegister($src3$$reg));
13012 %}
13013
13014 ins_pipe(pipe_class_default);
13015 %}
13016
13017 // src1 * (-src2) - src3
13018 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13019 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13020 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13021
13022 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13023
13024 ins_encode %{
13025 assert(UseFMA, "Needs FMA instructions support.");
13026 __ fnmaddd(as_FloatRegister($dst$$reg),
13027 as_FloatRegister($src1$$reg),
13028 as_FloatRegister($src2$$reg),
13029 as_FloatRegister($src3$$reg));
13030 %}
13031
13032 ins_pipe(pipe_class_default);
13033 %}
13034
13035 // src1 * src2 - src3
13036 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13037 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13038
13039 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13040
13041 ins_encode %{
13042 assert(UseFMA, "Needs FMA instructions support.");
13043 __ fnmsubs(as_FloatRegister($dst$$reg),
13044 as_FloatRegister($src1$$reg),
13045 as_FloatRegister($src2$$reg),
13046 as_FloatRegister($src3$$reg));
13047 %}
13048
13049 ins_pipe(pipe_class_default);
13050 %}
13051
13052 // src1 * src2 - src3
13053 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13054 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13055
13056 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13057
13058 ins_encode %{
13059 assert(UseFMA, "Needs FMA instructions support.");
13060 // n.b. insn name should be fnmsubd
13061 __ fnmsub(as_FloatRegister($dst$$reg),
13062 as_FloatRegister($src1$$reg),
13063 as_FloatRegister($src2$$reg),
13064 as_FloatRegister($src3$$reg));
13065 %}
13066
13067 ins_pipe(pipe_class_default);
13068 %}
13069
13070 // Math.max(HH)H (half-precision float)
13071 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13072 match(Set dst (MaxHF src1 src2));
13073 format %{ "fmaxh $dst, $src1, $src2" %}
13074 ins_encode %{
13075 __ fmaxh($dst$$FloatRegister,
13076 $src1$$FloatRegister,
13077 $src2$$FloatRegister);
13078 %}
13079 ins_pipe(fp_dop_reg_reg_s);
13080 %}
13081
13082 // Math.min(HH)H (half-precision float)
13083 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13084 match(Set dst (MinHF src1 src2));
13085 format %{ "fminh $dst, $src1, $src2" %}
13086 ins_encode %{
13087 __ fminh($dst$$FloatRegister,
13088 $src1$$FloatRegister,
13089 $src2$$FloatRegister);
13090 %}
13091 ins_pipe(fp_dop_reg_reg_s);
13092 %}
13093
13094 // Math.max(FF)F
13095 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13096 match(Set dst (MaxF src1 src2));
13097
13098 format %{ "fmaxs $dst, $src1, $src2" %}
13099 ins_encode %{
13100 __ fmaxs(as_FloatRegister($dst$$reg),
13101 as_FloatRegister($src1$$reg),
13102 as_FloatRegister($src2$$reg));
13103 %}
13104
13105 ins_pipe(fp_dop_reg_reg_s);
13106 %}
13107
13108 // Math.min(FF)F
13109 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13110 match(Set dst (MinF src1 src2));
13111
13112 format %{ "fmins $dst, $src1, $src2" %}
13113 ins_encode %{
13114 __ fmins(as_FloatRegister($dst$$reg),
13115 as_FloatRegister($src1$$reg),
13116 as_FloatRegister($src2$$reg));
13117 %}
13118
13119 ins_pipe(fp_dop_reg_reg_s);
13120 %}
13121
13122 // Math.max(DD)D
13123 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13124 match(Set dst (MaxD src1 src2));
13125
13126 format %{ "fmaxd $dst, $src1, $src2" %}
13127 ins_encode %{
13128 __ fmaxd(as_FloatRegister($dst$$reg),
13129 as_FloatRegister($src1$$reg),
13130 as_FloatRegister($src2$$reg));
13131 %}
13132
13133 ins_pipe(fp_dop_reg_reg_d);
13134 %}
13135
13136 // Math.min(DD)D
13137 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13138 match(Set dst (MinD src1 src2));
13139
13140 format %{ "fmind $dst, $src1, $src2" %}
13141 ins_encode %{
13142 __ fmind(as_FloatRegister($dst$$reg),
13143 as_FloatRegister($src1$$reg),
13144 as_FloatRegister($src2$$reg));
13145 %}
13146
13147 ins_pipe(fp_dop_reg_reg_d);
13148 %}
13149
13150 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13151 match(Set dst (DivHF src1 src2));
13152 format %{ "fdivh $dst, $src1, $src2" %}
13153 ins_encode %{
13154 __ fdivh($dst$$FloatRegister,
13155 $src1$$FloatRegister,
13156 $src2$$FloatRegister);
13157 %}
13158 ins_pipe(fp_div_s);
13159 %}
13160
13161 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13162 match(Set dst (DivF src1 src2));
13163
13164 ins_cost(INSN_COST * 18);
13165 format %{ "fdivs $dst, $src1, $src2" %}
13166
13167 ins_encode %{
13168 __ fdivs(as_FloatRegister($dst$$reg),
13169 as_FloatRegister($src1$$reg),
13170 as_FloatRegister($src2$$reg));
13171 %}
13172
13173 ins_pipe(fp_div_s);
13174 %}
13175
13176 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13177 match(Set dst (DivD src1 src2));
13178
13179 ins_cost(INSN_COST * 32);
13180 format %{ "fdivd $dst, $src1, $src2" %}
13181
13182 ins_encode %{
13183 __ fdivd(as_FloatRegister($dst$$reg),
13184 as_FloatRegister($src1$$reg),
13185 as_FloatRegister($src2$$reg));
13186 %}
13187
13188 ins_pipe(fp_div_d);
13189 %}
13190
13191 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13192 match(Set dst (NegF src));
13193
13194 ins_cost(INSN_COST * 3);
13195 format %{ "fneg $dst, $src" %}
13196
13197 ins_encode %{
13198 __ fnegs(as_FloatRegister($dst$$reg),
13199 as_FloatRegister($src$$reg));
13200 %}
13201
13202 ins_pipe(fp_uop_s);
13203 %}
13204
13205 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13206 match(Set dst (NegD src));
13207
13208 ins_cost(INSN_COST * 3);
13209 format %{ "fnegd $dst, $src" %}
13210
13211 ins_encode %{
13212 __ fnegd(as_FloatRegister($dst$$reg),
13213 as_FloatRegister($src$$reg));
13214 %}
13215
13216 ins_pipe(fp_uop_d);
13217 %}
13218
13219 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13220 %{
13221 match(Set dst (AbsI src));
13222
13223 effect(KILL cr);
13224 ins_cost(INSN_COST * 2);
13225 format %{ "cmpw $src, zr\n\t"
13226 "cnegw $dst, $src, Assembler::LT\t# int abs"
13227 %}
13228
13229 ins_encode %{
13230 __ cmpw(as_Register($src$$reg), zr);
13231 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13232 %}
13233 ins_pipe(pipe_class_default);
13234 %}
13235
13236 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13237 %{
13238 match(Set dst (AbsL src));
13239
13240 effect(KILL cr);
13241 ins_cost(INSN_COST * 2);
13242 format %{ "cmp $src, zr\n\t"
13243 "cneg $dst, $src, Assembler::LT\t# long abs"
13244 %}
13245
13246 ins_encode %{
13247 __ cmp(as_Register($src$$reg), zr);
13248 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13249 %}
13250 ins_pipe(pipe_class_default);
13251 %}
13252
13253 instruct absF_reg(vRegF dst, vRegF src) %{
13254 match(Set dst (AbsF src));
13255
13256 ins_cost(INSN_COST * 3);
13257 format %{ "fabss $dst, $src" %}
13258 ins_encode %{
13259 __ fabss(as_FloatRegister($dst$$reg),
13260 as_FloatRegister($src$$reg));
13261 %}
13262
13263 ins_pipe(fp_uop_s);
13264 %}
13265
13266 instruct absD_reg(vRegD dst, vRegD src) %{
13267 match(Set dst (AbsD src));
13268
13269 ins_cost(INSN_COST * 3);
13270 format %{ "fabsd $dst, $src" %}
13271 ins_encode %{
13272 __ fabsd(as_FloatRegister($dst$$reg),
13273 as_FloatRegister($src$$reg));
13274 %}
13275
13276 ins_pipe(fp_uop_d);
13277 %}
13278
13279 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13280 match(Set dst (AbsF (SubF src1 src2)));
13281
13282 ins_cost(INSN_COST * 3);
13283 format %{ "fabds $dst, $src1, $src2" %}
13284 ins_encode %{
13285 __ fabds(as_FloatRegister($dst$$reg),
13286 as_FloatRegister($src1$$reg),
13287 as_FloatRegister($src2$$reg));
13288 %}
13289
13290 ins_pipe(fp_uop_s);
13291 %}
13292
13293 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13294 match(Set dst (AbsD (SubD src1 src2)));
13295
13296 ins_cost(INSN_COST * 3);
13297 format %{ "fabdd $dst, $src1, $src2" %}
13298 ins_encode %{
13299 __ fabdd(as_FloatRegister($dst$$reg),
13300 as_FloatRegister($src1$$reg),
13301 as_FloatRegister($src2$$reg));
13302 %}
13303
13304 ins_pipe(fp_uop_d);
13305 %}
13306
13307 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13308 match(Set dst (SqrtD src));
13309
13310 ins_cost(INSN_COST * 50);
13311 format %{ "fsqrtd $dst, $src" %}
13312 ins_encode %{
13313 __ fsqrtd(as_FloatRegister($dst$$reg),
13314 as_FloatRegister($src$$reg));
13315 %}
13316
13317 ins_pipe(fp_div_s);
13318 %}
13319
13320 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13321 match(Set dst (SqrtF src));
13322
13323 ins_cost(INSN_COST * 50);
13324 format %{ "fsqrts $dst, $src" %}
13325 ins_encode %{
13326 __ fsqrts(as_FloatRegister($dst$$reg),
13327 as_FloatRegister($src$$reg));
13328 %}
13329
13330 ins_pipe(fp_div_d);
13331 %}
13332
13333 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13334 match(Set dst (SqrtHF src));
13335 format %{ "fsqrth $dst, $src" %}
13336 ins_encode %{
13337 __ fsqrth($dst$$FloatRegister,
13338 $src$$FloatRegister);
13339 %}
13340 ins_pipe(fp_div_s);
13341 %}
13342
13343 // Math.rint, floor, ceil
13344 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13345 match(Set dst (RoundDoubleMode src rmode));
13346 format %{ "frint $dst, $src, $rmode" %}
13347 ins_encode %{
13348 switch ($rmode$$constant) {
13349 case RoundDoubleModeNode::rmode_rint:
13350 __ frintnd(as_FloatRegister($dst$$reg),
13351 as_FloatRegister($src$$reg));
13352 break;
13353 case RoundDoubleModeNode::rmode_floor:
13354 __ frintmd(as_FloatRegister($dst$$reg),
13355 as_FloatRegister($src$$reg));
13356 break;
13357 case RoundDoubleModeNode::rmode_ceil:
13358 __ frintpd(as_FloatRegister($dst$$reg),
13359 as_FloatRegister($src$$reg));
13360 break;
13361 }
13362 %}
13363 ins_pipe(fp_uop_d);
13364 %}
13365
13366 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13367 match(Set dst (CopySignD src1 (Binary src2 zero)));
13368 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13369 format %{ "CopySignD $dst $src1 $src2" %}
13370 ins_encode %{
13371 FloatRegister dst = as_FloatRegister($dst$$reg),
13372 src1 = as_FloatRegister($src1$$reg),
13373 src2 = as_FloatRegister($src2$$reg),
13374 zero = as_FloatRegister($zero$$reg);
13375 __ fnegd(dst, zero);
13376 __ bsl(dst, __ T8B, src2, src1);
13377 %}
13378 ins_pipe(fp_uop_d);
13379 %}
13380
13381 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13382 match(Set dst (CopySignF src1 src2));
13383 effect(TEMP_DEF dst, USE src1, USE src2);
13384 format %{ "CopySignF $dst $src1 $src2" %}
13385 ins_encode %{
13386 FloatRegister dst = as_FloatRegister($dst$$reg),
13387 src1 = as_FloatRegister($src1$$reg),
13388 src2 = as_FloatRegister($src2$$reg);
13389 __ movi(dst, __ T2S, 0x80, 24);
13390 __ bsl(dst, __ T8B, src2, src1);
13391 %}
13392 ins_pipe(fp_uop_d);
13393 %}
13394
13395 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13396 match(Set dst (SignumD src (Binary zero one)));
13397 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13398 format %{ "signumD $dst, $src" %}
13399 ins_encode %{
13400 FloatRegister src = as_FloatRegister($src$$reg),
13401 dst = as_FloatRegister($dst$$reg),
13402 zero = as_FloatRegister($zero$$reg),
13403 one = as_FloatRegister($one$$reg);
13404 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13405 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13406 // Bit selection instruction gets bit from "one" for each enabled bit in
13407 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13408 // NaN the whole "src" will be copied because "dst" is zero. For all other
13409 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13410 // from "src", and all other bits are copied from 1.0.
13411 __ bsl(dst, __ T8B, one, src);
13412 %}
13413 ins_pipe(fp_uop_d);
13414 %}
13415
13416 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13417 match(Set dst (SignumF src (Binary zero one)));
13418 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13419 format %{ "signumF $dst, $src" %}
13420 ins_encode %{
13421 FloatRegister src = as_FloatRegister($src$$reg),
13422 dst = as_FloatRegister($dst$$reg),
13423 zero = as_FloatRegister($zero$$reg),
13424 one = as_FloatRegister($one$$reg);
13425 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13426 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13427 // Bit selection instruction gets bit from "one" for each enabled bit in
13428 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13429 // NaN the whole "src" will be copied because "dst" is zero. For all other
13430 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13431 // from "src", and all other bits are copied from 1.0.
13432 __ bsl(dst, __ T8B, one, src);
13433 %}
13434 ins_pipe(fp_uop_d);
13435 %}
13436
13437 instruct onspinwait() %{
13438 match(OnSpinWait);
13439 ins_cost(INSN_COST);
13440
13441 format %{ "onspinwait" %}
13442
13443 ins_encode %{
13444 __ spin_wait();
13445 %}
13446 ins_pipe(pipe_class_empty);
13447 %}
13448
13449 // ============================================================================
13450 // Logical Instructions
13451
13452 // Integer Logical Instructions
13453
13454 // And Instructions
13455
13456
13457 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13458 match(Set dst (AndI src1 src2));
13459
13460 format %{ "andw $dst, $src1, $src2\t# int" %}
13461
13462 ins_cost(INSN_COST);
13463 ins_encode %{
13464 __ andw(as_Register($dst$$reg),
13465 as_Register($src1$$reg),
13466 as_Register($src2$$reg));
13467 %}
13468
13469 ins_pipe(ialu_reg_reg);
13470 %}
13471
13472 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13473 match(Set dst (AndI src1 src2));
13474
13475 format %{ "andsw $dst, $src1, $src2\t# int" %}
13476
13477 ins_cost(INSN_COST);
13478 ins_encode %{
13479 __ andw(as_Register($dst$$reg),
13480 as_Register($src1$$reg),
13481 (uint64_t)($src2$$constant));
13482 %}
13483
13484 ins_pipe(ialu_reg_imm);
13485 %}
13486
13487 // Or Instructions
13488
13489 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13490 match(Set dst (OrI src1 src2));
13491
13492 format %{ "orrw $dst, $src1, $src2\t# int" %}
13493
13494 ins_cost(INSN_COST);
13495 ins_encode %{
13496 __ orrw(as_Register($dst$$reg),
13497 as_Register($src1$$reg),
13498 as_Register($src2$$reg));
13499 %}
13500
13501 ins_pipe(ialu_reg_reg);
13502 %}
13503
13504 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13505 match(Set dst (OrI src1 src2));
13506
13507 format %{ "orrw $dst, $src1, $src2\t# int" %}
13508
13509 ins_cost(INSN_COST);
13510 ins_encode %{
13511 __ orrw(as_Register($dst$$reg),
13512 as_Register($src1$$reg),
13513 (uint64_t)($src2$$constant));
13514 %}
13515
13516 ins_pipe(ialu_reg_imm);
13517 %}
13518
13519 // Xor Instructions
13520
13521 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13522 match(Set dst (XorI src1 src2));
13523
13524 format %{ "eorw $dst, $src1, $src2\t# int" %}
13525
13526 ins_cost(INSN_COST);
13527 ins_encode %{
13528 __ eorw(as_Register($dst$$reg),
13529 as_Register($src1$$reg),
13530 as_Register($src2$$reg));
13531 %}
13532
13533 ins_pipe(ialu_reg_reg);
13534 %}
13535
13536 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13537 match(Set dst (XorI src1 src2));
13538
13539 format %{ "eorw $dst, $src1, $src2\t# int" %}
13540
13541 ins_cost(INSN_COST);
13542 ins_encode %{
13543 __ eorw(as_Register($dst$$reg),
13544 as_Register($src1$$reg),
13545 (uint64_t)($src2$$constant));
13546 %}
13547
13548 ins_pipe(ialu_reg_imm);
13549 %}
13550
13551 // Long Logical Instructions
13552 // TODO
13553
13554 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13555 match(Set dst (AndL src1 src2));
13556
13557 format %{ "and $dst, $src1, $src2\t# int" %}
13558
13559 ins_cost(INSN_COST);
13560 ins_encode %{
13561 __ andr(as_Register($dst$$reg),
13562 as_Register($src1$$reg),
13563 as_Register($src2$$reg));
13564 %}
13565
13566 ins_pipe(ialu_reg_reg);
13567 %}
13568
13569 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13570 match(Set dst (AndL src1 src2));
13571
13572 format %{ "and $dst, $src1, $src2\t# int" %}
13573
13574 ins_cost(INSN_COST);
13575 ins_encode %{
13576 __ andr(as_Register($dst$$reg),
13577 as_Register($src1$$reg),
13578 (uint64_t)($src2$$constant));
13579 %}
13580
13581 ins_pipe(ialu_reg_imm);
13582 %}
13583
13584 // Or Instructions
13585
13586 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13587 match(Set dst (OrL src1 src2));
13588
13589 format %{ "orr $dst, $src1, $src2\t# int" %}
13590
13591 ins_cost(INSN_COST);
13592 ins_encode %{
13593 __ orr(as_Register($dst$$reg),
13594 as_Register($src1$$reg),
13595 as_Register($src2$$reg));
13596 %}
13597
13598 ins_pipe(ialu_reg_reg);
13599 %}
13600
13601 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13602 match(Set dst (OrL src1 src2));
13603
13604 format %{ "orr $dst, $src1, $src2\t# int" %}
13605
13606 ins_cost(INSN_COST);
13607 ins_encode %{
13608 __ orr(as_Register($dst$$reg),
13609 as_Register($src1$$reg),
13610 (uint64_t)($src2$$constant));
13611 %}
13612
13613 ins_pipe(ialu_reg_imm);
13614 %}
13615
13616 // Xor Instructions
13617
13618 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13619 match(Set dst (XorL src1 src2));
13620
13621 format %{ "eor $dst, $src1, $src2\t# int" %}
13622
13623 ins_cost(INSN_COST);
13624 ins_encode %{
13625 __ eor(as_Register($dst$$reg),
13626 as_Register($src1$$reg),
13627 as_Register($src2$$reg));
13628 %}
13629
13630 ins_pipe(ialu_reg_reg);
13631 %}
13632
13633 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13634 match(Set dst (XorL src1 src2));
13635
13636 ins_cost(INSN_COST);
13637 format %{ "eor $dst, $src1, $src2\t# int" %}
13638
13639 ins_encode %{
13640 __ eor(as_Register($dst$$reg),
13641 as_Register($src1$$reg),
13642 (uint64_t)($src2$$constant));
13643 %}
13644
13645 ins_pipe(ialu_reg_imm);
13646 %}
13647
13648 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13649 %{
13650 match(Set dst (ConvI2L src));
13651
13652 ins_cost(INSN_COST);
13653 format %{ "sxtw $dst, $src\t# i2l" %}
13654 ins_encode %{
13655 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13656 %}
13657 ins_pipe(ialu_reg_shift);
13658 %}
13659
13660 // this pattern occurs in bigmath arithmetic
13661 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13662 %{
13663 match(Set dst (AndL (ConvI2L src) mask));
13664
13665 ins_cost(INSN_COST);
13666 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13667 ins_encode %{
13668 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13669 %}
13670
13671 ins_pipe(ialu_reg_shift);
13672 %}
13673
13674 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13675 match(Set dst (ConvL2I src));
13676
13677 ins_cost(INSN_COST);
13678 format %{ "movw $dst, $src \t// l2i" %}
13679
13680 ins_encode %{
13681 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13682 %}
13683
13684 ins_pipe(ialu_reg);
13685 %}
13686
13687 instruct convD2F_reg(vRegF dst, vRegD src) %{
13688 match(Set dst (ConvD2F src));
13689
13690 ins_cost(INSN_COST * 5);
13691 format %{ "fcvtd $dst, $src \t// d2f" %}
13692
13693 ins_encode %{
13694 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13695 %}
13696
13697 ins_pipe(fp_d2f);
13698 %}
13699
13700 instruct convF2D_reg(vRegD dst, vRegF src) %{
13701 match(Set dst (ConvF2D src));
13702
13703 ins_cost(INSN_COST * 5);
13704 format %{ "fcvts $dst, $src \t// f2d" %}
13705
13706 ins_encode %{
13707 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13708 %}
13709
13710 ins_pipe(fp_f2d);
13711 %}
13712
13713 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13714 match(Set dst (ConvF2I src));
13715
13716 ins_cost(INSN_COST * 5);
13717 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13718
13719 ins_encode %{
13720 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13721 %}
13722
13723 ins_pipe(fp_f2i);
13724 %}
13725
13726 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13727 match(Set dst (ConvF2L src));
13728
13729 ins_cost(INSN_COST * 5);
13730 format %{ "fcvtzs $dst, $src \t// f2l" %}
13731
13732 ins_encode %{
13733 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13734 %}
13735
13736 ins_pipe(fp_f2l);
13737 %}
13738
13739 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13740 match(Set dst (ConvF2HF src));
13741 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13742 "smov $dst, $tmp\t# move result from $tmp to $dst"
13743 %}
13744 effect(TEMP tmp);
13745 ins_encode %{
13746 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13747 %}
13748 ins_pipe(pipe_slow);
13749 %}
13750
13751 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13752 match(Set dst (ConvHF2F src));
13753 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13754 "fcvt $dst, $tmp\t# convert half to single precision"
13755 %}
13756 effect(TEMP tmp);
13757 ins_encode %{
13758 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13759 %}
13760 ins_pipe(pipe_slow);
13761 %}
13762
13763 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13764 match(Set dst (ConvI2F src));
13765
13766 ins_cost(INSN_COST * 5);
13767 format %{ "scvtfws $dst, $src \t// i2f" %}
13768
13769 ins_encode %{
13770 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13771 %}
13772
13773 ins_pipe(fp_i2f);
13774 %}
13775
13776 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13777 match(Set dst (ConvL2F src));
13778
13779 ins_cost(INSN_COST * 5);
13780 format %{ "scvtfs $dst, $src \t// l2f" %}
13781
13782 ins_encode %{
13783 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13784 %}
13785
13786 ins_pipe(fp_l2f);
13787 %}
13788
13789 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13790 match(Set dst (ConvD2I src));
13791
13792 ins_cost(INSN_COST * 5);
13793 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13794
13795 ins_encode %{
13796 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13797 %}
13798
13799 ins_pipe(fp_d2i);
13800 %}
13801
13802 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13803 match(Set dst (ConvD2L src));
13804
13805 ins_cost(INSN_COST * 5);
13806 format %{ "fcvtzd $dst, $src \t// d2l" %}
13807
13808 ins_encode %{
13809 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13810 %}
13811
13812 ins_pipe(fp_d2l);
13813 %}
13814
13815 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13816 match(Set dst (ConvI2D src));
13817
13818 ins_cost(INSN_COST * 5);
13819 format %{ "scvtfwd $dst, $src \t// i2d" %}
13820
13821 ins_encode %{
13822 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13823 %}
13824
13825 ins_pipe(fp_i2d);
13826 %}
13827
13828 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13829 match(Set dst (ConvL2D src));
13830
13831 ins_cost(INSN_COST * 5);
13832 format %{ "scvtfd $dst, $src \t// l2d" %}
13833
13834 ins_encode %{
13835 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13836 %}
13837
13838 ins_pipe(fp_l2d);
13839 %}
13840
13841 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13842 %{
13843 match(Set dst (RoundD src));
13844 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13845 format %{ "java_round_double $dst,$src"%}
13846 ins_encode %{
13847 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13848 as_FloatRegister($ftmp$$reg));
13849 %}
13850 ins_pipe(pipe_slow);
13851 %}
13852
13853 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13854 %{
13855 match(Set dst (RoundF src));
13856 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13857 format %{ "java_round_float $dst,$src"%}
13858 ins_encode %{
13859 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13860 as_FloatRegister($ftmp$$reg));
13861 %}
13862 ins_pipe(pipe_slow);
13863 %}
13864
13865 // stack <-> reg and reg <-> reg shuffles with no conversion
13866
13867 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13868
13869 match(Set dst (MoveF2I src));
13870
13871 effect(DEF dst, USE src);
13872
13873 ins_cost(4 * INSN_COST);
13874
13875 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13876
13877 ins_encode %{
13878 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13879 %}
13880
13881 ins_pipe(iload_reg_reg);
13882
13883 %}
13884
13885 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13886
13887 match(Set dst (MoveI2F src));
13888
13889 effect(DEF dst, USE src);
13890
13891 ins_cost(4 * INSN_COST);
13892
13893 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13894
13895 ins_encode %{
13896 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13897 %}
13898
13899 ins_pipe(pipe_class_memory);
13900
13901 %}
13902
13903 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13904
13905 match(Set dst (MoveD2L src));
13906
13907 effect(DEF dst, USE src);
13908
13909 ins_cost(4 * INSN_COST);
13910
13911 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13912
13913 ins_encode %{
13914 __ ldr($dst$$Register, Address(sp, $src$$disp));
13915 %}
13916
13917 ins_pipe(iload_reg_reg);
13918
13919 %}
13920
13921 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13922
13923 match(Set dst (MoveL2D src));
13924
13925 effect(DEF dst, USE src);
13926
13927 ins_cost(4 * INSN_COST);
13928
13929 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13930
13931 ins_encode %{
13932 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13933 %}
13934
13935 ins_pipe(pipe_class_memory);
13936
13937 %}
13938
13939 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13940
13941 match(Set dst (MoveF2I src));
13942
13943 effect(DEF dst, USE src);
13944
13945 ins_cost(INSN_COST);
13946
13947 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13948
13949 ins_encode %{
13950 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13951 %}
13952
13953 ins_pipe(pipe_class_memory);
13954
13955 %}
13956
13957 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13958
13959 match(Set dst (MoveI2F src));
13960
13961 effect(DEF dst, USE src);
13962
13963 ins_cost(INSN_COST);
13964
13965 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13966
13967 ins_encode %{
13968 __ strw($src$$Register, Address(sp, $dst$$disp));
13969 %}
13970
13971 ins_pipe(istore_reg_reg);
13972
13973 %}
13974
13975 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13976
13977 match(Set dst (MoveD2L src));
13978
13979 effect(DEF dst, USE src);
13980
13981 ins_cost(INSN_COST);
13982
13983 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13984
13985 ins_encode %{
13986 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13987 %}
13988
13989 ins_pipe(pipe_class_memory);
13990
13991 %}
13992
13993 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13994
13995 match(Set dst (MoveL2D src));
13996
13997 effect(DEF dst, USE src);
13998
13999 ins_cost(INSN_COST);
14000
14001 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14002
14003 ins_encode %{
14004 __ str($src$$Register, Address(sp, $dst$$disp));
14005 %}
14006
14007 ins_pipe(istore_reg_reg);
14008
14009 %}
14010
14011 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14012
14013 match(Set dst (MoveF2I src));
14014
14015 effect(DEF dst, USE src);
14016
14017 ins_cost(INSN_COST);
14018
14019 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14020
14021 ins_encode %{
14022 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14023 %}
14024
14025 ins_pipe(fp_f2i);
14026
14027 %}
14028
14029 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14030
14031 match(Set dst (MoveI2F src));
14032
14033 effect(DEF dst, USE src);
14034
14035 ins_cost(INSN_COST);
14036
14037 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14038
14039 ins_encode %{
14040 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14041 %}
14042
14043 ins_pipe(fp_i2f);
14044
14045 %}
14046
14047 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14048
14049 match(Set dst (MoveD2L src));
14050
14051 effect(DEF dst, USE src);
14052
14053 ins_cost(INSN_COST);
14054
14055 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14056
14057 ins_encode %{
14058 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14059 %}
14060
14061 ins_pipe(fp_d2l);
14062
14063 %}
14064
14065 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14066
14067 match(Set dst (MoveL2D src));
14068
14069 effect(DEF dst, USE src);
14070
14071 ins_cost(INSN_COST);
14072
14073 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14074
14075 ins_encode %{
14076 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14077 %}
14078
14079 ins_pipe(fp_l2d);
14080
14081 %}
14082
14083 // ============================================================================
14084 // clearing of an array
14085
14086 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14087 %{
14088 match(Set dummy (ClearArray cnt base));
14089 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14090
14091 ins_cost(4 * INSN_COST);
14092 format %{ "ClearArray $cnt, $base" %}
14093
14094 ins_encode %{
14095 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14096 if (tpc == nullptr) {
14097 ciEnv::current()->record_failure("CodeCache is full");
14098 return;
14099 }
14100 %}
14101
14102 ins_pipe(pipe_class_memory);
14103 %}
14104
14105 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14106 %{
14107 predicate((uint64_t)n->in(2)->get_long()
14108 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14109 match(Set dummy (ClearArray cnt base));
14110 effect(TEMP temp, USE_KILL base, KILL cr);
14111
14112 ins_cost(4 * INSN_COST);
14113 format %{ "ClearArray $cnt, $base" %}
14114
14115 ins_encode %{
14116 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14117 if (tpc == nullptr) {
14118 ciEnv::current()->record_failure("CodeCache is full");
14119 return;
14120 }
14121 %}
14122
14123 ins_pipe(pipe_class_memory);
14124 %}
14125
14126 // ============================================================================
14127 // Overflow Math Instructions
14128
14129 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14130 %{
14131 match(Set cr (OverflowAddI op1 op2));
14132
14133 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14134 ins_cost(INSN_COST);
14135 ins_encode %{
14136 __ cmnw($op1$$Register, $op2$$Register);
14137 %}
14138
14139 ins_pipe(icmp_reg_reg);
14140 %}
14141
14142 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14143 %{
14144 match(Set cr (OverflowAddI op1 op2));
14145
14146 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14147 ins_cost(INSN_COST);
14148 ins_encode %{
14149 __ cmnw($op1$$Register, $op2$$constant);
14150 %}
14151
14152 ins_pipe(icmp_reg_imm);
14153 %}
14154
14155 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14156 %{
14157 match(Set cr (OverflowAddL op1 op2));
14158
14159 format %{ "cmn $op1, $op2\t# overflow check long" %}
14160 ins_cost(INSN_COST);
14161 ins_encode %{
14162 __ cmn($op1$$Register, $op2$$Register);
14163 %}
14164
14165 ins_pipe(icmp_reg_reg);
14166 %}
14167
14168 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14169 %{
14170 match(Set cr (OverflowAddL op1 op2));
14171
14172 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14173 ins_cost(INSN_COST);
14174 ins_encode %{
14175 __ adds(zr, $op1$$Register, $op2$$constant);
14176 %}
14177
14178 ins_pipe(icmp_reg_imm);
14179 %}
14180
14181 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14182 %{
14183 match(Set cr (OverflowSubI op1 op2));
14184
14185 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14186 ins_cost(INSN_COST);
14187 ins_encode %{
14188 __ cmpw($op1$$Register, $op2$$Register);
14189 %}
14190
14191 ins_pipe(icmp_reg_reg);
14192 %}
14193
14194 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14195 %{
14196 match(Set cr (OverflowSubI op1 op2));
14197
14198 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14199 ins_cost(INSN_COST);
14200 ins_encode %{
14201 __ cmpw($op1$$Register, $op2$$constant);
14202 %}
14203
14204 ins_pipe(icmp_reg_imm);
14205 %}
14206
14207 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14208 %{
14209 match(Set cr (OverflowSubL op1 op2));
14210
14211 format %{ "cmp $op1, $op2\t# overflow check long" %}
14212 ins_cost(INSN_COST);
14213 ins_encode %{
14214 __ cmp($op1$$Register, $op2$$Register);
14215 %}
14216
14217 ins_pipe(icmp_reg_reg);
14218 %}
14219
14220 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14221 %{
14222 match(Set cr (OverflowSubL op1 op2));
14223
14224 format %{ "cmp $op1, $op2\t# overflow check long" %}
14225 ins_cost(INSN_COST);
14226 ins_encode %{
14227 __ subs(zr, $op1$$Register, $op2$$constant);
14228 %}
14229
14230 ins_pipe(icmp_reg_imm);
14231 %}
14232
14233 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14234 %{
14235 match(Set cr (OverflowSubI zero op1));
14236
14237 format %{ "cmpw zr, $op1\t# overflow check int" %}
14238 ins_cost(INSN_COST);
14239 ins_encode %{
14240 __ cmpw(zr, $op1$$Register);
14241 %}
14242
14243 ins_pipe(icmp_reg_imm);
14244 %}
14245
14246 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14247 %{
14248 match(Set cr (OverflowSubL zero op1));
14249
14250 format %{ "cmp zr, $op1\t# overflow check long" %}
14251 ins_cost(INSN_COST);
14252 ins_encode %{
14253 __ cmp(zr, $op1$$Register);
14254 %}
14255
14256 ins_pipe(icmp_reg_imm);
14257 %}
14258
14259 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14260 %{
14261 match(Set cr (OverflowMulI op1 op2));
14262
14263 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14264 "cmp rscratch1, rscratch1, sxtw\n\t"
14265 "movw rscratch1, #0x80000000\n\t"
14266 "cselw rscratch1, rscratch1, zr, NE\n\t"
14267 "cmpw rscratch1, #1" %}
14268 ins_cost(5 * INSN_COST);
14269 ins_encode %{
14270 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14271 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14272 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14273 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14274 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14275 %}
14276
14277 ins_pipe(pipe_slow);
14278 %}
14279
14280 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14281 %{
14282 match(If cmp (OverflowMulI op1 op2));
14283 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14284 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14285 effect(USE labl, KILL cr);
14286
14287 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14288 "cmp rscratch1, rscratch1, sxtw\n\t"
14289 "b$cmp $labl" %}
14290 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14291 ins_encode %{
14292 Label* L = $labl$$label;
14293 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14294 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14295 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14296 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14297 %}
14298
14299 ins_pipe(pipe_serial);
14300 %}
14301
14302 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14303 %{
14304 match(Set cr (OverflowMulL op1 op2));
14305
14306 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14307 "smulh rscratch2, $op1, $op2\n\t"
14308 "cmp rscratch2, rscratch1, ASR #63\n\t"
14309 "movw rscratch1, #0x80000000\n\t"
14310 "cselw rscratch1, rscratch1, zr, NE\n\t"
14311 "cmpw rscratch1, #1" %}
14312 ins_cost(6 * INSN_COST);
14313 ins_encode %{
14314 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14315 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14316 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14317 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14318 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14319 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14320 %}
14321
14322 ins_pipe(pipe_slow);
14323 %}
14324
14325 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14326 %{
14327 match(If cmp (OverflowMulL op1 op2));
14328 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14329 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14330 effect(USE labl, KILL cr);
14331
14332 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14333 "smulh rscratch2, $op1, $op2\n\t"
14334 "cmp rscratch2, rscratch1, ASR #63\n\t"
14335 "b$cmp $labl" %}
14336 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14337 ins_encode %{
14338 Label* L = $labl$$label;
14339 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14340 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14341 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14342 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14343 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14344 %}
14345
14346 ins_pipe(pipe_serial);
14347 %}
14348
14349 // ============================================================================
14350 // Compare Instructions
14351
14352 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14353 %{
14354 match(Set cr (CmpI op1 op2));
14355
14356 effect(DEF cr, USE op1, USE op2);
14357
14358 ins_cost(INSN_COST);
14359 format %{ "cmpw $op1, $op2" %}
14360
14361 ins_encode(aarch64_enc_cmpw(op1, op2));
14362
14363 ins_pipe(icmp_reg_reg);
14364 %}
14365
14366 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14367 %{
14368 match(Set cr (CmpI op1 zero));
14369
14370 effect(DEF cr, USE op1);
14371
14372 ins_cost(INSN_COST);
14373 format %{ "cmpw $op1, 0" %}
14374
14375 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14376
14377 ins_pipe(icmp_reg_imm);
14378 %}
14379
14380 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14381 %{
14382 match(Set cr (CmpI op1 op2));
14383
14384 effect(DEF cr, USE op1);
14385
14386 ins_cost(INSN_COST);
14387 format %{ "cmpw $op1, $op2" %}
14388
14389 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14390
14391 ins_pipe(icmp_reg_imm);
14392 %}
14393
14394 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14395 %{
14396 match(Set cr (CmpI op1 op2));
14397
14398 effect(DEF cr, USE op1);
14399
14400 ins_cost(INSN_COST * 2);
14401 format %{ "cmpw $op1, $op2" %}
14402
14403 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14404
14405 ins_pipe(icmp_reg_imm);
14406 %}
14407
14408 // Unsigned compare Instructions; really, same as signed compare
14409 // except it should only be used to feed an If or a CMovI which takes a
14410 // cmpOpU.
14411
14412 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14413 %{
14414 match(Set cr (CmpU op1 op2));
14415
14416 effect(DEF cr, USE op1, USE op2);
14417
14418 ins_cost(INSN_COST);
14419 format %{ "cmpw $op1, $op2\t# unsigned" %}
14420
14421 ins_encode(aarch64_enc_cmpw(op1, op2));
14422
14423 ins_pipe(icmp_reg_reg);
14424 %}
14425
14426 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14427 %{
14428 match(Set cr (CmpU op1 zero));
14429
14430 effect(DEF cr, USE op1);
14431
14432 ins_cost(INSN_COST);
14433 format %{ "cmpw $op1, #0\t# unsigned" %}
14434
14435 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14436
14437 ins_pipe(icmp_reg_imm);
14438 %}
14439
14440 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14441 %{
14442 match(Set cr (CmpU op1 op2));
14443
14444 effect(DEF cr, USE op1);
14445
14446 ins_cost(INSN_COST);
14447 format %{ "cmpw $op1, $op2\t# unsigned" %}
14448
14449 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14450
14451 ins_pipe(icmp_reg_imm);
14452 %}
14453
14454 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14455 %{
14456 match(Set cr (CmpU op1 op2));
14457
14458 effect(DEF cr, USE op1);
14459
14460 ins_cost(INSN_COST * 2);
14461 format %{ "cmpw $op1, $op2\t# unsigned" %}
14462
14463 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14464
14465 ins_pipe(icmp_reg_imm);
14466 %}
14467
14468 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14469 %{
14470 match(Set cr (CmpL op1 op2));
14471
14472 effect(DEF cr, USE op1, USE op2);
14473
14474 ins_cost(INSN_COST);
14475 format %{ "cmp $op1, $op2" %}
14476
14477 ins_encode(aarch64_enc_cmp(op1, op2));
14478
14479 ins_pipe(icmp_reg_reg);
14480 %}
14481
14482 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14483 %{
14484 match(Set cr (CmpL op1 zero));
14485
14486 effect(DEF cr, USE op1);
14487
14488 ins_cost(INSN_COST);
14489 format %{ "tst $op1" %}
14490
14491 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14492
14493 ins_pipe(icmp_reg_imm);
14494 %}
14495
14496 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14497 %{
14498 match(Set cr (CmpL op1 op2));
14499
14500 effect(DEF cr, USE op1);
14501
14502 ins_cost(INSN_COST);
14503 format %{ "cmp $op1, $op2" %}
14504
14505 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14506
14507 ins_pipe(icmp_reg_imm);
14508 %}
14509
14510 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14511 %{
14512 match(Set cr (CmpL op1 op2));
14513
14514 effect(DEF cr, USE op1);
14515
14516 ins_cost(INSN_COST * 2);
14517 format %{ "cmp $op1, $op2" %}
14518
14519 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14520
14521 ins_pipe(icmp_reg_imm);
14522 %}
14523
14524 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14525 %{
14526 match(Set cr (CmpUL op1 op2));
14527
14528 effect(DEF cr, USE op1, USE op2);
14529
14530 ins_cost(INSN_COST);
14531 format %{ "cmp $op1, $op2" %}
14532
14533 ins_encode(aarch64_enc_cmp(op1, op2));
14534
14535 ins_pipe(icmp_reg_reg);
14536 %}
14537
14538 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14539 %{
14540 match(Set cr (CmpUL op1 zero));
14541
14542 effect(DEF cr, USE op1);
14543
14544 ins_cost(INSN_COST);
14545 format %{ "tst $op1" %}
14546
14547 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14548
14549 ins_pipe(icmp_reg_imm);
14550 %}
14551
14552 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14553 %{
14554 match(Set cr (CmpUL op1 op2));
14555
14556 effect(DEF cr, USE op1);
14557
14558 ins_cost(INSN_COST);
14559 format %{ "cmp $op1, $op2" %}
14560
14561 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14562
14563 ins_pipe(icmp_reg_imm);
14564 %}
14565
14566 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14567 %{
14568 match(Set cr (CmpUL op1 op2));
14569
14570 effect(DEF cr, USE op1);
14571
14572 ins_cost(INSN_COST * 2);
14573 format %{ "cmp $op1, $op2" %}
14574
14575 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14576
14577 ins_pipe(icmp_reg_imm);
14578 %}
14579
14580 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14581 %{
14582 match(Set cr (CmpP op1 op2));
14583
14584 effect(DEF cr, USE op1, USE op2);
14585
14586 ins_cost(INSN_COST);
14587 format %{ "cmp $op1, $op2\t // ptr" %}
14588
14589 ins_encode(aarch64_enc_cmpp(op1, op2));
14590
14591 ins_pipe(icmp_reg_reg);
14592 %}
14593
14594 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14595 %{
14596 match(Set cr (CmpN op1 op2));
14597
14598 effect(DEF cr, USE op1, USE op2);
14599
14600 ins_cost(INSN_COST);
14601 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14602
14603 ins_encode(aarch64_enc_cmpn(op1, op2));
14604
14605 ins_pipe(icmp_reg_reg);
14606 %}
14607
14608 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14609 %{
14610 match(Set cr (CmpP op1 zero));
14611
14612 effect(DEF cr, USE op1, USE zero);
14613
14614 ins_cost(INSN_COST);
14615 format %{ "cmp $op1, 0\t // ptr" %}
14616
14617 ins_encode(aarch64_enc_testp(op1));
14618
14619 ins_pipe(icmp_reg_imm);
14620 %}
14621
14622 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14623 %{
14624 match(Set cr (CmpN op1 zero));
14625
14626 effect(DEF cr, USE op1, USE zero);
14627
14628 ins_cost(INSN_COST);
14629 format %{ "cmp $op1, 0\t // compressed ptr" %}
14630
14631 ins_encode(aarch64_enc_testn(op1));
14632
14633 ins_pipe(icmp_reg_imm);
14634 %}
14635
14636 // FP comparisons
14637 //
14638 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14639 // using normal cmpOp. See declaration of rFlagsReg for details.
14640
14641 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14642 %{
14643 match(Set cr (CmpF src1 src2));
14644
14645 ins_cost(3 * INSN_COST);
14646 format %{ "fcmps $src1, $src2" %}
14647
14648 ins_encode %{
14649 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14650 %}
14651
14652 ins_pipe(pipe_class_compare);
14653 %}
14654
14655 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14656 %{
14657 match(Set cr (CmpF src1 src2));
14658
14659 ins_cost(3 * INSN_COST);
14660 format %{ "fcmps $src1, 0.0" %}
14661
14662 ins_encode %{
14663 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14664 %}
14665
14666 ins_pipe(pipe_class_compare);
14667 %}
14668 // FROM HERE
14669
14670 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14671 %{
14672 match(Set cr (CmpD src1 src2));
14673
14674 ins_cost(3 * INSN_COST);
14675 format %{ "fcmpd $src1, $src2" %}
14676
14677 ins_encode %{
14678 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14679 %}
14680
14681 ins_pipe(pipe_class_compare);
14682 %}
14683
14684 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14685 %{
14686 match(Set cr (CmpD src1 src2));
14687
14688 ins_cost(3 * INSN_COST);
14689 format %{ "fcmpd $src1, 0.0" %}
14690
14691 ins_encode %{
14692 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14693 %}
14694
14695 ins_pipe(pipe_class_compare);
14696 %}
14697
14698 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14699 %{
14700 match(Set dst (CmpF3 src1 src2));
14701 effect(KILL cr);
14702
14703 ins_cost(5 * INSN_COST);
14704 format %{ "fcmps $src1, $src2\n\t"
14705 "csinvw($dst, zr, zr, eq\n\t"
14706 "csnegw($dst, $dst, $dst, lt)"
14707 %}
14708
14709 ins_encode %{
14710 Label done;
14711 FloatRegister s1 = as_FloatRegister($src1$$reg);
14712 FloatRegister s2 = as_FloatRegister($src2$$reg);
14713 Register d = as_Register($dst$$reg);
14714 __ fcmps(s1, s2);
14715 // installs 0 if EQ else -1
14716 __ csinvw(d, zr, zr, Assembler::EQ);
14717 // keeps -1 if less or unordered else installs 1
14718 __ csnegw(d, d, d, Assembler::LT);
14719 __ bind(done);
14720 %}
14721
14722 ins_pipe(pipe_class_default);
14723
14724 %}
14725
14726 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14727 %{
14728 match(Set dst (CmpD3 src1 src2));
14729 effect(KILL cr);
14730
14731 ins_cost(5 * INSN_COST);
14732 format %{ "fcmpd $src1, $src2\n\t"
14733 "csinvw($dst, zr, zr, eq\n\t"
14734 "csnegw($dst, $dst, $dst, lt)"
14735 %}
14736
14737 ins_encode %{
14738 Label done;
14739 FloatRegister s1 = as_FloatRegister($src1$$reg);
14740 FloatRegister s2 = as_FloatRegister($src2$$reg);
14741 Register d = as_Register($dst$$reg);
14742 __ fcmpd(s1, s2);
14743 // installs 0 if EQ else -1
14744 __ csinvw(d, zr, zr, Assembler::EQ);
14745 // keeps -1 if less or unordered else installs 1
14746 __ csnegw(d, d, d, Assembler::LT);
14747 __ bind(done);
14748 %}
14749 ins_pipe(pipe_class_default);
14750
14751 %}
14752
14753 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14754 %{
14755 match(Set dst (CmpF3 src1 zero));
14756 effect(KILL cr);
14757
14758 ins_cost(5 * INSN_COST);
14759 format %{ "fcmps $src1, 0.0\n\t"
14760 "csinvw($dst, zr, zr, eq\n\t"
14761 "csnegw($dst, $dst, $dst, lt)"
14762 %}
14763
14764 ins_encode %{
14765 Label done;
14766 FloatRegister s1 = as_FloatRegister($src1$$reg);
14767 Register d = as_Register($dst$$reg);
14768 __ fcmps(s1, 0.0);
14769 // installs 0 if EQ else -1
14770 __ csinvw(d, zr, zr, Assembler::EQ);
14771 // keeps -1 if less or unordered else installs 1
14772 __ csnegw(d, d, d, Assembler::LT);
14773 __ bind(done);
14774 %}
14775
14776 ins_pipe(pipe_class_default);
14777
14778 %}
14779
14780 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14781 %{
14782 match(Set dst (CmpD3 src1 zero));
14783 effect(KILL cr);
14784
14785 ins_cost(5 * INSN_COST);
14786 format %{ "fcmpd $src1, 0.0\n\t"
14787 "csinvw($dst, zr, zr, eq\n\t"
14788 "csnegw($dst, $dst, $dst, lt)"
14789 %}
14790
14791 ins_encode %{
14792 Label done;
14793 FloatRegister s1 = as_FloatRegister($src1$$reg);
14794 Register d = as_Register($dst$$reg);
14795 __ fcmpd(s1, 0.0);
14796 // installs 0 if EQ else -1
14797 __ csinvw(d, zr, zr, Assembler::EQ);
14798 // keeps -1 if less or unordered else installs 1
14799 __ csnegw(d, d, d, Assembler::LT);
14800 __ bind(done);
14801 %}
14802 ins_pipe(pipe_class_default);
14803
14804 %}
14805
14806 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14807 %{
14808 match(Set dst (CmpLTMask p q));
14809 effect(KILL cr);
14810
14811 ins_cost(3 * INSN_COST);
14812
14813 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14814 "csetw $dst, lt\n\t"
14815 "subw $dst, zr, $dst"
14816 %}
14817
14818 ins_encode %{
14819 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14820 __ csetw(as_Register($dst$$reg), Assembler::LT);
14821 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14822 %}
14823
14824 ins_pipe(ialu_reg_reg);
14825 %}
14826
14827 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14828 %{
14829 match(Set dst (CmpLTMask src zero));
14830 effect(KILL cr);
14831
14832 ins_cost(INSN_COST);
14833
14834 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14835
14836 ins_encode %{
14837 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14838 %}
14839
14840 ins_pipe(ialu_reg_shift);
14841 %}
14842
14843 // ============================================================================
14844 // Max and Min
14845
14846 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14847
14848 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14849 %{
14850 effect(DEF cr, USE src);
14851 ins_cost(INSN_COST);
14852 format %{ "cmpw $src, 0" %}
14853
14854 ins_encode %{
14855 __ cmpw($src$$Register, 0);
14856 %}
14857 ins_pipe(icmp_reg_imm);
14858 %}
14859
14860 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14861 %{
14862 match(Set dst (MinI src1 src2));
14863 ins_cost(INSN_COST * 3);
14864
14865 expand %{
14866 rFlagsReg cr;
14867 compI_reg_reg(cr, src1, src2);
14868 cmovI_reg_reg_lt(dst, src1, src2, cr);
14869 %}
14870 %}
14871
14872 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14873 %{
14874 match(Set dst (MaxI src1 src2));
14875 ins_cost(INSN_COST * 3);
14876
14877 expand %{
14878 rFlagsReg cr;
14879 compI_reg_reg(cr, src1, src2);
14880 cmovI_reg_reg_gt(dst, src1, src2, cr);
14881 %}
14882 %}
14883
14884
14885 // ============================================================================
14886 // Branch Instructions
14887
14888 // Direct Branch.
14889 instruct branch(label lbl)
14890 %{
14891 match(Goto);
14892
14893 effect(USE lbl);
14894
14895 ins_cost(BRANCH_COST);
14896 format %{ "b $lbl" %}
14897
14898 ins_encode(aarch64_enc_b(lbl));
14899
14900 ins_pipe(pipe_branch);
14901 %}
14902
14903 // Conditional Near Branch
14904 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14905 %{
14906 // Same match rule as `branchConFar'.
14907 match(If cmp cr);
14908
14909 effect(USE lbl);
14910
14911 ins_cost(BRANCH_COST);
14912 // If set to 1 this indicates that the current instruction is a
14913 // short variant of a long branch. This avoids using this
14914 // instruction in first-pass matching. It will then only be used in
14915 // the `Shorten_branches' pass.
14916 // ins_short_branch(1);
14917 format %{ "b$cmp $lbl" %}
14918
14919 ins_encode(aarch64_enc_br_con(cmp, lbl));
14920
14921 ins_pipe(pipe_branch_cond);
14922 %}
14923
14924 // Conditional Near Branch Unsigned
14925 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14926 %{
14927 // Same match rule as `branchConFar'.
14928 match(If cmp cr);
14929
14930 effect(USE lbl);
14931
14932 ins_cost(BRANCH_COST);
14933 // If set to 1 this indicates that the current instruction is a
14934 // short variant of a long branch. This avoids using this
14935 // instruction in first-pass matching. It will then only be used in
14936 // the `Shorten_branches' pass.
14937 // ins_short_branch(1);
14938 format %{ "b$cmp $lbl\t# unsigned" %}
14939
14940 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14941
14942 ins_pipe(pipe_branch_cond);
14943 %}
14944
14945 // Make use of CBZ and CBNZ. These instructions, as well as being
14946 // shorter than (cmp; branch), have the additional benefit of not
14947 // killing the flags.
14948
14949 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14950 match(If cmp (CmpI op1 op2));
14951 effect(USE labl);
14952
14953 ins_cost(BRANCH_COST);
14954 format %{ "cbw$cmp $op1, $labl" %}
14955 ins_encode %{
14956 Label* L = $labl$$label;
14957 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14958 if (cond == Assembler::EQ)
14959 __ cbzw($op1$$Register, *L);
14960 else
14961 __ cbnzw($op1$$Register, *L);
14962 %}
14963 ins_pipe(pipe_cmp_branch);
14964 %}
14965
14966 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14967 match(If cmp (CmpL op1 op2));
14968 effect(USE labl);
14969
14970 ins_cost(BRANCH_COST);
14971 format %{ "cb$cmp $op1, $labl" %}
14972 ins_encode %{
14973 Label* L = $labl$$label;
14974 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14975 if (cond == Assembler::EQ)
14976 __ cbz($op1$$Register, *L);
14977 else
14978 __ cbnz($op1$$Register, *L);
14979 %}
14980 ins_pipe(pipe_cmp_branch);
14981 %}
14982
14983 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14984 match(If cmp (CmpP op1 op2));
14985 effect(USE labl);
14986
14987 ins_cost(BRANCH_COST);
14988 format %{ "cb$cmp $op1, $labl" %}
14989 ins_encode %{
14990 Label* L = $labl$$label;
14991 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14992 if (cond == Assembler::EQ)
14993 __ cbz($op1$$Register, *L);
14994 else
14995 __ cbnz($op1$$Register, *L);
14996 %}
14997 ins_pipe(pipe_cmp_branch);
14998 %}
14999
15000 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15001 match(If cmp (CmpN op1 op2));
15002 effect(USE labl);
15003
15004 ins_cost(BRANCH_COST);
15005 format %{ "cbw$cmp $op1, $labl" %}
15006 ins_encode %{
15007 Label* L = $labl$$label;
15008 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15009 if (cond == Assembler::EQ)
15010 __ cbzw($op1$$Register, *L);
15011 else
15012 __ cbnzw($op1$$Register, *L);
15013 %}
15014 ins_pipe(pipe_cmp_branch);
15015 %}
15016
15017 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15018 match(If cmp (CmpP (DecodeN oop) zero));
15019 effect(USE labl);
15020
15021 ins_cost(BRANCH_COST);
15022 format %{ "cb$cmp $oop, $labl" %}
15023 ins_encode %{
15024 Label* L = $labl$$label;
15025 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15026 if (cond == Assembler::EQ)
15027 __ cbzw($oop$$Register, *L);
15028 else
15029 __ cbnzw($oop$$Register, *L);
15030 %}
15031 ins_pipe(pipe_cmp_branch);
15032 %}
15033
15034 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15035 match(If cmp (CmpU op1 op2));
15036 effect(USE labl);
15037
15038 ins_cost(BRANCH_COST);
15039 format %{ "cbw$cmp $op1, $labl" %}
15040 ins_encode %{
15041 Label* L = $labl$$label;
15042 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15043 if (cond == Assembler::EQ || cond == Assembler::LS) {
15044 __ cbzw($op1$$Register, *L);
15045 } else {
15046 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15047 __ cbnzw($op1$$Register, *L);
15048 }
15049 %}
15050 ins_pipe(pipe_cmp_branch);
15051 %}
15052
15053 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15054 match(If cmp (CmpUL op1 op2));
15055 effect(USE labl);
15056
15057 ins_cost(BRANCH_COST);
15058 format %{ "cb$cmp $op1, $labl" %}
15059 ins_encode %{
15060 Label* L = $labl$$label;
15061 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15062 if (cond == Assembler::EQ || cond == Assembler::LS) {
15063 __ cbz($op1$$Register, *L);
15064 } else {
15065 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15066 __ cbnz($op1$$Register, *L);
15067 }
15068 %}
15069 ins_pipe(pipe_cmp_branch);
15070 %}
15071
15072 // Test bit and Branch
15073
15074 // Patterns for short (< 32KiB) variants
15075 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15076 match(If cmp (CmpL op1 op2));
15077 effect(USE labl);
15078
15079 ins_cost(BRANCH_COST);
15080 format %{ "cb$cmp $op1, $labl # long" %}
15081 ins_encode %{
15082 Label* L = $labl$$label;
15083 Assembler::Condition cond =
15084 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15085 __ tbr(cond, $op1$$Register, 63, *L);
15086 %}
15087 ins_pipe(pipe_cmp_branch);
15088 ins_short_branch(1);
15089 %}
15090
15091 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15092 match(If cmp (CmpI op1 op2));
15093 effect(USE labl);
15094
15095 ins_cost(BRANCH_COST);
15096 format %{ "cb$cmp $op1, $labl # int" %}
15097 ins_encode %{
15098 Label* L = $labl$$label;
15099 Assembler::Condition cond =
15100 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15101 __ tbr(cond, $op1$$Register, 31, *L);
15102 %}
15103 ins_pipe(pipe_cmp_branch);
15104 ins_short_branch(1);
15105 %}
15106
15107 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15108 match(If cmp (CmpL (AndL op1 op2) op3));
15109 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15110 effect(USE labl);
15111
15112 ins_cost(BRANCH_COST);
15113 format %{ "tb$cmp $op1, $op2, $labl" %}
15114 ins_encode %{
15115 Label* L = $labl$$label;
15116 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15117 int bit = exact_log2_long($op2$$constant);
15118 __ tbr(cond, $op1$$Register, bit, *L);
15119 %}
15120 ins_pipe(pipe_cmp_branch);
15121 ins_short_branch(1);
15122 %}
15123
15124 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15125 match(If cmp (CmpI (AndI op1 op2) op3));
15126 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15127 effect(USE labl);
15128
15129 ins_cost(BRANCH_COST);
15130 format %{ "tb$cmp $op1, $op2, $labl" %}
15131 ins_encode %{
15132 Label* L = $labl$$label;
15133 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15134 int bit = exact_log2((juint)$op2$$constant);
15135 __ tbr(cond, $op1$$Register, bit, *L);
15136 %}
15137 ins_pipe(pipe_cmp_branch);
15138 ins_short_branch(1);
15139 %}
15140
15141 // And far variants
15142 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15143 match(If cmp (CmpL op1 op2));
15144 effect(USE labl);
15145
15146 ins_cost(BRANCH_COST);
15147 format %{ "cb$cmp $op1, $labl # long" %}
15148 ins_encode %{
15149 Label* L = $labl$$label;
15150 Assembler::Condition cond =
15151 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15152 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15153 %}
15154 ins_pipe(pipe_cmp_branch);
15155 %}
15156
15157 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15158 match(If cmp (CmpI op1 op2));
15159 effect(USE labl);
15160
15161 ins_cost(BRANCH_COST);
15162 format %{ "cb$cmp $op1, $labl # int" %}
15163 ins_encode %{
15164 Label* L = $labl$$label;
15165 Assembler::Condition cond =
15166 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15167 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15168 %}
15169 ins_pipe(pipe_cmp_branch);
15170 %}
15171
15172 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15173 match(If cmp (CmpL (AndL op1 op2) op3));
15174 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15175 effect(USE labl);
15176
15177 ins_cost(BRANCH_COST);
15178 format %{ "tb$cmp $op1, $op2, $labl" %}
15179 ins_encode %{
15180 Label* L = $labl$$label;
15181 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15182 int bit = exact_log2_long($op2$$constant);
15183 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15184 %}
15185 ins_pipe(pipe_cmp_branch);
15186 %}
15187
15188 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15189 match(If cmp (CmpI (AndI op1 op2) op3));
15190 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15191 effect(USE labl);
15192
15193 ins_cost(BRANCH_COST);
15194 format %{ "tb$cmp $op1, $op2, $labl" %}
15195 ins_encode %{
15196 Label* L = $labl$$label;
15197 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15198 int bit = exact_log2((juint)$op2$$constant);
15199 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15200 %}
15201 ins_pipe(pipe_cmp_branch);
15202 %}
15203
15204 // Test bits
15205
15206 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15207 match(Set cr (CmpL (AndL op1 op2) op3));
15208 predicate(Assembler::operand_valid_for_logical_immediate
15209 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15210
15211 ins_cost(INSN_COST);
15212 format %{ "tst $op1, $op2 # long" %}
15213 ins_encode %{
15214 __ tst($op1$$Register, $op2$$constant);
15215 %}
15216 ins_pipe(ialu_reg_reg);
15217 %}
15218
15219 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15220 match(Set cr (CmpI (AndI op1 op2) op3));
15221 predicate(Assembler::operand_valid_for_logical_immediate
15222 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15223
15224 ins_cost(INSN_COST);
15225 format %{ "tst $op1, $op2 # int" %}
15226 ins_encode %{
15227 __ tstw($op1$$Register, $op2$$constant);
15228 %}
15229 ins_pipe(ialu_reg_reg);
15230 %}
15231
15232 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15233 match(Set cr (CmpL (AndL op1 op2) op3));
15234
15235 ins_cost(INSN_COST);
15236 format %{ "tst $op1, $op2 # long" %}
15237 ins_encode %{
15238 __ tst($op1$$Register, $op2$$Register);
15239 %}
15240 ins_pipe(ialu_reg_reg);
15241 %}
15242
15243 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15244 match(Set cr (CmpI (AndI op1 op2) op3));
15245
15246 ins_cost(INSN_COST);
15247 format %{ "tstw $op1, $op2 # int" %}
15248 ins_encode %{
15249 __ tstw($op1$$Register, $op2$$Register);
15250 %}
15251 ins_pipe(ialu_reg_reg);
15252 %}
15253
15254
15255 // Conditional Far Branch
15256 // Conditional Far Branch Unsigned
15257 // TODO: fixme
15258
15259 // counted loop end branch near
15260 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15261 %{
15262 match(CountedLoopEnd cmp cr);
15263
15264 effect(USE lbl);
15265
15266 ins_cost(BRANCH_COST);
15267 // short variant.
15268 // ins_short_branch(1);
15269 format %{ "b$cmp $lbl \t// counted loop end" %}
15270
15271 ins_encode(aarch64_enc_br_con(cmp, lbl));
15272
15273 ins_pipe(pipe_branch);
15274 %}
15275
15276 // counted loop end branch far
15277 // TODO: fixme
15278
15279 // ============================================================================
15280 // inlined locking and unlocking
15281
15282 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15283 %{
15284 match(Set cr (FastLock object box));
15285 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15286
15287 ins_cost(5 * INSN_COST);
15288 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15289
15290 ins_encode %{
15291 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15292 %}
15293
15294 ins_pipe(pipe_serial);
15295 %}
15296
15297 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15298 %{
15299 match(Set cr (FastUnlock object box));
15300 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15301
15302 ins_cost(5 * INSN_COST);
15303 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15304
15305 ins_encode %{
15306 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15307 %}
15308
15309 ins_pipe(pipe_serial);
15310 %}
15311
15312 // ============================================================================
15313 // Safepoint Instructions
15314
15315 // TODO
15316 // provide a near and far version of this code
15317
15318 instruct safePoint(rFlagsReg cr, iRegP poll)
15319 %{
15320 match(SafePoint poll);
15321 effect(KILL cr);
15322
15323 format %{
15324 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15325 %}
15326 ins_encode %{
15327 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15328 %}
15329 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15330 %}
15331
15332
15333 // ============================================================================
15334 // Procedure Call/Return Instructions
15335
15336 // Call Java Static Instruction
15337
15338 instruct CallStaticJavaDirect(method meth)
15339 %{
15340 match(CallStaticJava);
15341
15342 effect(USE meth);
15343
15344 ins_cost(CALL_COST);
15345
15346 format %{ "call,static $meth \t// ==> " %}
15347
15348 ins_encode(aarch64_enc_java_static_call(meth),
15349 aarch64_enc_call_epilog);
15350
15351 ins_pipe(pipe_class_call);
15352 %}
15353
15354 // TO HERE
15355
15356 // Call Java Dynamic Instruction
15357 instruct CallDynamicJavaDirect(method meth)
15358 %{
15359 match(CallDynamicJava);
15360
15361 effect(USE meth);
15362
15363 ins_cost(CALL_COST);
15364
15365 format %{ "CALL,dynamic $meth \t// ==> " %}
15366
15367 ins_encode(aarch64_enc_java_dynamic_call(meth),
15368 aarch64_enc_call_epilog);
15369
15370 ins_pipe(pipe_class_call);
15371 %}
15372
15373 // Call Runtime Instruction
15374
15375 instruct CallRuntimeDirect(method meth)
15376 %{
15377 match(CallRuntime);
15378
15379 effect(USE meth);
15380
15381 ins_cost(CALL_COST);
15382
15383 format %{ "CALL, runtime $meth" %}
15384
15385 ins_encode( aarch64_enc_java_to_runtime(meth) );
15386
15387 ins_pipe(pipe_class_call);
15388 %}
15389
15390 // Call Runtime Instruction
15391
15392 instruct CallLeafDirect(method meth)
15393 %{
15394 match(CallLeaf);
15395
15396 effect(USE meth);
15397
15398 ins_cost(CALL_COST);
15399
15400 format %{ "CALL, runtime leaf $meth" %}
15401
15402 ins_encode( aarch64_enc_java_to_runtime(meth) );
15403
15404 ins_pipe(pipe_class_call);
15405 %}
15406
15407 // Call Runtime Instruction without safepoint and with vector arguments
15408 instruct CallLeafDirectVector(method meth)
15409 %{
15410 match(CallLeafVector);
15411
15412 effect(USE meth);
15413
15414 ins_cost(CALL_COST);
15415
15416 format %{ "CALL, runtime leaf vector $meth" %}
15417
15418 ins_encode(aarch64_enc_java_to_runtime(meth));
15419
15420 ins_pipe(pipe_class_call);
15421 %}
15422
15423 // Call Runtime Instruction
15424
15425 instruct CallLeafNoFPDirect(method meth)
15426 %{
15427 match(CallLeafNoFP);
15428
15429 effect(USE meth);
15430
15431 ins_cost(CALL_COST);
15432
15433 format %{ "CALL, runtime leaf nofp $meth" %}
15434
15435 ins_encode( aarch64_enc_java_to_runtime(meth) );
15436
15437 ins_pipe(pipe_class_call);
15438 %}
15439
15440 // Tail Call; Jump from runtime stub to Java code.
15441 // Also known as an 'interprocedural jump'.
15442 // Target of jump will eventually return to caller.
15443 // TailJump below removes the return address.
15444 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15445 // emitted just above the TailCall which has reset rfp to the caller state.
15446 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15447 %{
15448 match(TailCall jump_target method_ptr);
15449
15450 ins_cost(CALL_COST);
15451
15452 format %{ "br $jump_target\t# $method_ptr holds method" %}
15453
15454 ins_encode(aarch64_enc_tail_call(jump_target));
15455
15456 ins_pipe(pipe_class_call);
15457 %}
15458
15459 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15460 %{
15461 match(TailJump jump_target ex_oop);
15462
15463 ins_cost(CALL_COST);
15464
15465 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15466
15467 ins_encode(aarch64_enc_tail_jmp(jump_target));
15468
15469 ins_pipe(pipe_class_call);
15470 %}
15471
15472 // Forward exception.
15473 instruct ForwardExceptionjmp()
15474 %{
15475 match(ForwardException);
15476 ins_cost(CALL_COST);
15477
15478 format %{ "b forward_exception_stub" %}
15479 ins_encode %{
15480 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15481 %}
15482 ins_pipe(pipe_class_call);
15483 %}
15484
15485 // Create exception oop: created by stack-crawling runtime code.
15486 // Created exception is now available to this handler, and is setup
15487 // just prior to jumping to this handler. No code emitted.
15488 // TODO check
15489 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15490 instruct CreateException(iRegP_R0 ex_oop)
15491 %{
15492 match(Set ex_oop (CreateEx));
15493
15494 format %{ " -- \t// exception oop; no code emitted" %}
15495
15496 size(0);
15497
15498 ins_encode( /*empty*/ );
15499
15500 ins_pipe(pipe_class_empty);
15501 %}
15502
15503 // Rethrow exception: The exception oop will come in the first
15504 // argument position. Then JUMP (not call) to the rethrow stub code.
15505 instruct RethrowException() %{
15506 match(Rethrow);
15507 ins_cost(CALL_COST);
15508
15509 format %{ "b rethrow_stub" %}
15510
15511 ins_encode( aarch64_enc_rethrow() );
15512
15513 ins_pipe(pipe_class_call);
15514 %}
15515
15516
15517 // Return Instruction
15518 // epilog node loads ret address into lr as part of frame pop
15519 instruct Ret()
15520 %{
15521 match(Return);
15522
15523 format %{ "ret\t// return register" %}
15524
15525 ins_encode( aarch64_enc_ret() );
15526
15527 ins_pipe(pipe_branch);
15528 %}
15529
15530 // Die now.
15531 instruct ShouldNotReachHere() %{
15532 match(Halt);
15533
15534 ins_cost(CALL_COST);
15535 format %{ "ShouldNotReachHere" %}
15536
15537 ins_encode %{
15538 if (is_reachable()) {
15539 const char* str = __ code_string(_halt_reason);
15540 __ stop(str);
15541 }
15542 %}
15543
15544 ins_pipe(pipe_class_default);
15545 %}
15546
15547 // ============================================================================
15548 // Partial Subtype Check
15549 //
15550 // superklass array for an instance of the superklass. Set a hidden
15551 // internal cache on a hit (cache is checked with exposed code in
15552 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15553 // encoding ALSO sets flags.
15554
15555 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15556 %{
15557 match(Set result (PartialSubtypeCheck sub super));
15558 predicate(!UseSecondarySupersTable);
15559 effect(KILL cr, KILL temp);
15560
15561 ins_cost(20 * INSN_COST); // slightly larger than the next version
15562 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15563
15564 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15565
15566 opcode(0x1); // Force zero of result reg on hit
15567
15568 ins_pipe(pipe_class_memory);
15569 %}
15570
15571 // Two versions of partialSubtypeCheck, both used when we need to
15572 // search for a super class in the secondary supers array. The first
15573 // is used when we don't know _a priori_ the class being searched
15574 // for. The second, far more common, is used when we do know: this is
15575 // used for instanceof, checkcast, and any case where C2 can determine
15576 // it by constant propagation.
15577
15578 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15579 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15580 rFlagsReg cr)
15581 %{
15582 match(Set result (PartialSubtypeCheck sub super));
15583 predicate(UseSecondarySupersTable);
15584 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15585
15586 ins_cost(10 * INSN_COST); // slightly larger than the next version
15587 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15588
15589 ins_encode %{
15590 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15591 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15592 $vtemp$$FloatRegister,
15593 $result$$Register, /*L_success*/nullptr);
15594 %}
15595
15596 ins_pipe(pipe_class_memory);
15597 %}
15598
15599 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15600 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15601 rFlagsReg cr)
15602 %{
15603 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15604 predicate(UseSecondarySupersTable);
15605 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15606
15607 ins_cost(5 * INSN_COST); // smaller than the next version
15608 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15609
15610 ins_encode %{
15611 bool success = false;
15612 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15613 if (InlineSecondarySupersTest) {
15614 success =
15615 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15616 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15617 $vtemp$$FloatRegister,
15618 $result$$Register,
15619 super_klass_slot);
15620 } else {
15621 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15622 success = (call != nullptr);
15623 }
15624 if (!success) {
15625 ciEnv::current()->record_failure("CodeCache is full");
15626 return;
15627 }
15628 %}
15629
15630 ins_pipe(pipe_class_memory);
15631 %}
15632
15633 // Intrisics for String.compareTo()
15634
15635 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15636 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15637 %{
15638 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15639 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15640 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15641
15642 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15643 ins_encode %{
15644 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15645 __ string_compare($str1$$Register, $str2$$Register,
15646 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15647 $tmp1$$Register, $tmp2$$Register,
15648 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15649 %}
15650 ins_pipe(pipe_class_memory);
15651 %}
15652
15653 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15654 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15655 %{
15656 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15657 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15658 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15659
15660 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15661 ins_encode %{
15662 __ string_compare($str1$$Register, $str2$$Register,
15663 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15664 $tmp1$$Register, $tmp2$$Register,
15665 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15666 %}
15667 ins_pipe(pipe_class_memory);
15668 %}
15669
15670 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15671 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15672 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15673 %{
15674 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15675 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15676 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15677 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15678
15679 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15680 ins_encode %{
15681 __ string_compare($str1$$Register, $str2$$Register,
15682 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15683 $tmp1$$Register, $tmp2$$Register,
15684 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15685 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15686 %}
15687 ins_pipe(pipe_class_memory);
15688 %}
15689
15690 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15691 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15692 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15693 %{
15694 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15695 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15696 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15697 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15698
15699 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15700 ins_encode %{
15701 __ string_compare($str1$$Register, $str2$$Register,
15702 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15703 $tmp1$$Register, $tmp2$$Register,
15704 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15705 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15706 %}
15707 ins_pipe(pipe_class_memory);
15708 %}
15709
15710 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15711 // these string_compare variants as NEON register type for convenience so that the prototype of
15712 // string_compare can be shared with all variants.
15713
15714 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15715 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15716 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15717 pRegGov_P1 pgtmp2, rFlagsReg cr)
15718 %{
15719 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15720 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15721 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15722 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15723
15724 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15725 ins_encode %{
15726 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15727 __ string_compare($str1$$Register, $str2$$Register,
15728 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15729 $tmp1$$Register, $tmp2$$Register,
15730 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15731 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15732 StrIntrinsicNode::LL);
15733 %}
15734 ins_pipe(pipe_class_memory);
15735 %}
15736
15737 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15738 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15739 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15740 pRegGov_P1 pgtmp2, rFlagsReg cr)
15741 %{
15742 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15743 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15744 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15745 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15746
15747 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15748 ins_encode %{
15749 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15750 __ string_compare($str1$$Register, $str2$$Register,
15751 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15752 $tmp1$$Register, $tmp2$$Register,
15753 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15754 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15755 StrIntrinsicNode::LU);
15756 %}
15757 ins_pipe(pipe_class_memory);
15758 %}
15759
15760 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15761 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15762 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15763 pRegGov_P1 pgtmp2, rFlagsReg cr)
15764 %{
15765 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15766 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15767 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15768 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15769
15770 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15771 ins_encode %{
15772 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15773 __ string_compare($str1$$Register, $str2$$Register,
15774 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15775 $tmp1$$Register, $tmp2$$Register,
15776 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15777 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15778 StrIntrinsicNode::UL);
15779 %}
15780 ins_pipe(pipe_class_memory);
15781 %}
15782
15783 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15784 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15785 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15786 pRegGov_P1 pgtmp2, rFlagsReg cr)
15787 %{
15788 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15789 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15790 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15791 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15792
15793 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15794 ins_encode %{
15795 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15796 __ string_compare($str1$$Register, $str2$$Register,
15797 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15798 $tmp1$$Register, $tmp2$$Register,
15799 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15800 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15801 StrIntrinsicNode::UU);
15802 %}
15803 ins_pipe(pipe_class_memory);
15804 %}
15805
15806 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15807 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15808 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15809 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15810 %{
15811 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15812 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15813 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15814 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15815 TEMP vtmp0, TEMP vtmp1, KILL cr);
15816 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15817 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15818
15819 ins_encode %{
15820 __ string_indexof($str1$$Register, $str2$$Register,
15821 $cnt1$$Register, $cnt2$$Register,
15822 $tmp1$$Register, $tmp2$$Register,
15823 $tmp3$$Register, $tmp4$$Register,
15824 $tmp5$$Register, $tmp6$$Register,
15825 -1, $result$$Register, StrIntrinsicNode::UU);
15826 %}
15827 ins_pipe(pipe_class_memory);
15828 %}
15829
15830 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15831 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15832 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15833 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15834 %{
15835 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15836 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15837 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15838 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15839 TEMP vtmp0, TEMP vtmp1, KILL cr);
15840 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15841 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15842
15843 ins_encode %{
15844 __ string_indexof($str1$$Register, $str2$$Register,
15845 $cnt1$$Register, $cnt2$$Register,
15846 $tmp1$$Register, $tmp2$$Register,
15847 $tmp3$$Register, $tmp4$$Register,
15848 $tmp5$$Register, $tmp6$$Register,
15849 -1, $result$$Register, StrIntrinsicNode::LL);
15850 %}
15851 ins_pipe(pipe_class_memory);
15852 %}
15853
15854 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15855 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15856 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15857 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15858 %{
15859 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15860 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15861 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15862 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15863 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15864 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15865 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15866
15867 ins_encode %{
15868 __ string_indexof($str1$$Register, $str2$$Register,
15869 $cnt1$$Register, $cnt2$$Register,
15870 $tmp1$$Register, $tmp2$$Register,
15871 $tmp3$$Register, $tmp4$$Register,
15872 $tmp5$$Register, $tmp6$$Register,
15873 -1, $result$$Register, StrIntrinsicNode::UL);
15874 %}
15875 ins_pipe(pipe_class_memory);
15876 %}
15877
15878 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15879 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15880 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15881 %{
15882 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15883 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15884 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15885 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15886 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15887 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15888
15889 ins_encode %{
15890 int icnt2 = (int)$int_cnt2$$constant;
15891 __ string_indexof($str1$$Register, $str2$$Register,
15892 $cnt1$$Register, zr,
15893 $tmp1$$Register, $tmp2$$Register,
15894 $tmp3$$Register, $tmp4$$Register, zr, zr,
15895 icnt2, $result$$Register, StrIntrinsicNode::UU);
15896 %}
15897 ins_pipe(pipe_class_memory);
15898 %}
15899
15900 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15901 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15902 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15903 %{
15904 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15905 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15906 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15907 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15908 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15909 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15910
15911 ins_encode %{
15912 int icnt2 = (int)$int_cnt2$$constant;
15913 __ string_indexof($str1$$Register, $str2$$Register,
15914 $cnt1$$Register, zr,
15915 $tmp1$$Register, $tmp2$$Register,
15916 $tmp3$$Register, $tmp4$$Register, zr, zr,
15917 icnt2, $result$$Register, StrIntrinsicNode::LL);
15918 %}
15919 ins_pipe(pipe_class_memory);
15920 %}
15921
15922 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15923 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15924 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15925 %{
15926 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15927 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15928 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15929 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15930 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15931 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15932
15933 ins_encode %{
15934 int icnt2 = (int)$int_cnt2$$constant;
15935 __ string_indexof($str1$$Register, $str2$$Register,
15936 $cnt1$$Register, zr,
15937 $tmp1$$Register, $tmp2$$Register,
15938 $tmp3$$Register, $tmp4$$Register, zr, zr,
15939 icnt2, $result$$Register, StrIntrinsicNode::UL);
15940 %}
15941 ins_pipe(pipe_class_memory);
15942 %}
15943
15944 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15945 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15946 iRegINoSp tmp3, rFlagsReg cr)
15947 %{
15948 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15949 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15950 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15951 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15952
15953 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15954
15955 ins_encode %{
15956 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15957 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15958 $tmp3$$Register);
15959 %}
15960 ins_pipe(pipe_class_memory);
15961 %}
15962
15963 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15964 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15965 iRegINoSp tmp3, rFlagsReg cr)
15966 %{
15967 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15968 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15969 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15970 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15971
15972 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15973
15974 ins_encode %{
15975 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15976 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15977 $tmp3$$Register);
15978 %}
15979 ins_pipe(pipe_class_memory);
15980 %}
15981
15982 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15983 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15984 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15985 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15986 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15987 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15988 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15989 ins_encode %{
15990 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15991 $result$$Register, $ztmp1$$FloatRegister,
15992 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15993 $ptmp$$PRegister, true /* isL */);
15994 %}
15995 ins_pipe(pipe_class_memory);
15996 %}
15997
15998 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15999 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16000 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16001 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16002 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16003 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16004 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16005 ins_encode %{
16006 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16007 $result$$Register, $ztmp1$$FloatRegister,
16008 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16009 $ptmp$$PRegister, false /* isL */);
16010 %}
16011 ins_pipe(pipe_class_memory);
16012 %}
16013
16014 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16015 iRegI_R0 result, rFlagsReg cr)
16016 %{
16017 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16018 match(Set result (StrEquals (Binary str1 str2) cnt));
16019 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16020
16021 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16022 ins_encode %{
16023 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16024 __ string_equals($str1$$Register, $str2$$Register,
16025 $result$$Register, $cnt$$Register);
16026 %}
16027 ins_pipe(pipe_class_memory);
16028 %}
16029
16030 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16031 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16032 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16033 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16034 iRegP_R10 tmp, rFlagsReg cr)
16035 %{
16036 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16037 match(Set result (AryEq ary1 ary2));
16038 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16039 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16040 TEMP vtmp6, TEMP vtmp7, KILL cr);
16041
16042 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16043 ins_encode %{
16044 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16045 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16046 $result$$Register, $tmp$$Register, 1);
16047 if (tpc == nullptr) {
16048 ciEnv::current()->record_failure("CodeCache is full");
16049 return;
16050 }
16051 %}
16052 ins_pipe(pipe_class_memory);
16053 %}
16054
16055 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16056 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16057 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16058 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16059 iRegP_R10 tmp, rFlagsReg cr)
16060 %{
16061 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16062 match(Set result (AryEq ary1 ary2));
16063 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16064 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16065 TEMP vtmp6, TEMP vtmp7, KILL cr);
16066
16067 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16068 ins_encode %{
16069 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16070 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16071 $result$$Register, $tmp$$Register, 2);
16072 if (tpc == nullptr) {
16073 ciEnv::current()->record_failure("CodeCache is full");
16074 return;
16075 }
16076 %}
16077 ins_pipe(pipe_class_memory);
16078 %}
16079
16080 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16081 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16082 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16083 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16084 %{
16085 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16086 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16087 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16088
16089 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16090 ins_encode %{
16091 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16092 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16093 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16094 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16095 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16096 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16097 (BasicType)$basic_type$$constant);
16098 if (tpc == nullptr) {
16099 ciEnv::current()->record_failure("CodeCache is full");
16100 return;
16101 }
16102 %}
16103 ins_pipe(pipe_class_memory);
16104 %}
16105
16106 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16107 %{
16108 match(Set result (CountPositives ary1 len));
16109 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16110 format %{ "count positives byte[] $ary1,$len -> $result" %}
16111 ins_encode %{
16112 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16113 if (tpc == nullptr) {
16114 ciEnv::current()->record_failure("CodeCache is full");
16115 return;
16116 }
16117 %}
16118 ins_pipe( pipe_slow );
16119 %}
16120
16121 // fast char[] to byte[] compression
16122 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16123 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16124 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16125 iRegI_R0 result, rFlagsReg cr)
16126 %{
16127 match(Set result (StrCompressedCopy src (Binary dst len)));
16128 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16129 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16130
16131 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16132 ins_encode %{
16133 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16134 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16135 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16136 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16137 %}
16138 ins_pipe(pipe_slow);
16139 %}
16140
16141 // fast byte[] to char[] inflation
16142 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16143 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16144 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16145 %{
16146 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16147 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16148 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16149 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16150
16151 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16152 ins_encode %{
16153 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16154 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16155 $vtmp2$$FloatRegister, $tmp$$Register);
16156 if (tpc == nullptr) {
16157 ciEnv::current()->record_failure("CodeCache is full");
16158 return;
16159 }
16160 %}
16161 ins_pipe(pipe_class_memory);
16162 %}
16163
16164 // encode char[] to byte[] in ISO_8859_1
16165 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16166 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16167 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16168 iRegI_R0 result, rFlagsReg cr)
16169 %{
16170 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16171 match(Set result (EncodeISOArray src (Binary dst len)));
16172 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16173 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16174
16175 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16176 ins_encode %{
16177 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16178 $result$$Register, false,
16179 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16180 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16181 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16182 %}
16183 ins_pipe(pipe_class_memory);
16184 %}
16185
16186 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16187 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16188 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16189 iRegI_R0 result, rFlagsReg cr)
16190 %{
16191 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16192 match(Set result (EncodeISOArray src (Binary dst len)));
16193 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16194 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16195
16196 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16197 ins_encode %{
16198 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16199 $result$$Register, true,
16200 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16201 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16202 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16203 %}
16204 ins_pipe(pipe_class_memory);
16205 %}
16206
16207 //----------------------------- CompressBits/ExpandBits ------------------------
16208
16209 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16210 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16211 match(Set dst (CompressBits src mask));
16212 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16213 format %{ "mov $tsrc, $src\n\t"
16214 "mov $tmask, $mask\n\t"
16215 "bext $tdst, $tsrc, $tmask\n\t"
16216 "mov $dst, $tdst"
16217 %}
16218 ins_encode %{
16219 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16220 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16221 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16222 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16223 %}
16224 ins_pipe(pipe_slow);
16225 %}
16226
16227 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16228 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16229 match(Set dst (CompressBits (LoadI mem) mask));
16230 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16231 format %{ "ldrs $tsrc, $mem\n\t"
16232 "ldrs $tmask, $mask\n\t"
16233 "bext $tdst, $tsrc, $tmask\n\t"
16234 "mov $dst, $tdst"
16235 %}
16236 ins_encode %{
16237 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16238 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16239 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16240 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16241 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16242 %}
16243 ins_pipe(pipe_slow);
16244 %}
16245
16246 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16247 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16248 match(Set dst (CompressBits src mask));
16249 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16250 format %{ "mov $tsrc, $src\n\t"
16251 "mov $tmask, $mask\n\t"
16252 "bext $tdst, $tsrc, $tmask\n\t"
16253 "mov $dst, $tdst"
16254 %}
16255 ins_encode %{
16256 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16257 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16258 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16259 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16260 %}
16261 ins_pipe(pipe_slow);
16262 %}
16263
16264 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16265 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16266 match(Set dst (CompressBits (LoadL mem) mask));
16267 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16268 format %{ "ldrd $tsrc, $mem\n\t"
16269 "ldrd $tmask, $mask\n\t"
16270 "bext $tdst, $tsrc, $tmask\n\t"
16271 "mov $dst, $tdst"
16272 %}
16273 ins_encode %{
16274 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16275 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16276 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16277 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16278 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16279 %}
16280 ins_pipe(pipe_slow);
16281 %}
16282
16283 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16284 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16285 match(Set dst (ExpandBits src mask));
16286 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16287 format %{ "mov $tsrc, $src\n\t"
16288 "mov $tmask, $mask\n\t"
16289 "bdep $tdst, $tsrc, $tmask\n\t"
16290 "mov $dst, $tdst"
16291 %}
16292 ins_encode %{
16293 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16294 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16295 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16296 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16297 %}
16298 ins_pipe(pipe_slow);
16299 %}
16300
16301 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16302 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16303 match(Set dst (ExpandBits (LoadI mem) mask));
16304 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16305 format %{ "ldrs $tsrc, $mem\n\t"
16306 "ldrs $tmask, $mask\n\t"
16307 "bdep $tdst, $tsrc, $tmask\n\t"
16308 "mov $dst, $tdst"
16309 %}
16310 ins_encode %{
16311 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16312 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16313 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16314 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16315 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16316 %}
16317 ins_pipe(pipe_slow);
16318 %}
16319
16320 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16321 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16322 match(Set dst (ExpandBits src mask));
16323 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16324 format %{ "mov $tsrc, $src\n\t"
16325 "mov $tmask, $mask\n\t"
16326 "bdep $tdst, $tsrc, $tmask\n\t"
16327 "mov $dst, $tdst"
16328 %}
16329 ins_encode %{
16330 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16331 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16332 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16333 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16334 %}
16335 ins_pipe(pipe_slow);
16336 %}
16337
16338
16339 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16340 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16341 match(Set dst (ExpandBits (LoadL mem) mask));
16342 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16343 format %{ "ldrd $tsrc, $mem\n\t"
16344 "ldrd $tmask, $mask\n\t"
16345 "bdep $tdst, $tsrc, $tmask\n\t"
16346 "mov $dst, $tdst"
16347 %}
16348 ins_encode %{
16349 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16350 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16351 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16352 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16353 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16354 %}
16355 ins_pipe(pipe_slow);
16356 %}
16357
16358 //----------------------------- Reinterpret ----------------------------------
16359 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16360 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16361 match(Set dst (ReinterpretHF2S src));
16362 format %{ "reinterpretHF2S $dst, $src" %}
16363 ins_encode %{
16364 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16365 %}
16366 ins_pipe(pipe_slow);
16367 %}
16368
16369 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16370 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16371 match(Set dst (ReinterpretS2HF src));
16372 format %{ "reinterpretS2HF $dst, $src" %}
16373 ins_encode %{
16374 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16375 %}
16376 ins_pipe(pipe_slow);
16377 %}
16378
16379 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16380 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16381 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16382 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16383 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16384 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16385 // can be omitted in this pattern, resulting in -
16386 // fcvt $dst, $src // Convert float to half-precision float
16387 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16388 %{
16389 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16390 format %{ "convF2HFAndS2HF $dst, $src" %}
16391 ins_encode %{
16392 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16393 %}
16394 ins_pipe(pipe_slow);
16395 %}
16396
16397 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16398 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16399 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16400 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16401 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16402 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16403 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16404 // resulting in -
16405 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16406 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16407 %{
16408 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16409 format %{ "convHF2SAndHF2F $dst, $src" %}
16410 ins_encode %{
16411 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16412 %}
16413 ins_pipe(pipe_slow);
16414 %}
16415
16416 // ============================================================================
16417 // This name is KNOWN by the ADLC and cannot be changed.
16418 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16419 // for this guy.
16420 instruct tlsLoadP(thread_RegP dst)
16421 %{
16422 match(Set dst (ThreadLocal));
16423
16424 ins_cost(0);
16425
16426 format %{ " -- \t// $dst=Thread::current(), empty" %}
16427
16428 size(0);
16429
16430 ins_encode( /*empty*/ );
16431
16432 ins_pipe(pipe_class_empty);
16433 %}
16434
16435 //----------PEEPHOLE RULES-----------------------------------------------------
16436 // These must follow all instruction definitions as they use the names
16437 // defined in the instructions definitions.
16438 //
16439 // peepmatch ( root_instr_name [preceding_instruction]* );
16440 //
16441 // peepconstraint %{
16442 // (instruction_number.operand_name relational_op instruction_number.operand_name
16443 // [, ...] );
16444 // // instruction numbers are zero-based using left to right order in peepmatch
16445 //
16446 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16447 // // provide an instruction_number.operand_name for each operand that appears
16448 // // in the replacement instruction's match rule
16449 //
16450 // ---------VM FLAGS---------------------------------------------------------
16451 //
16452 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16453 //
16454 // Each peephole rule is given an identifying number starting with zero and
16455 // increasing by one in the order seen by the parser. An individual peephole
16456 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16457 // on the command-line.
16458 //
16459 // ---------CURRENT LIMITATIONS----------------------------------------------
16460 //
16461 // Only match adjacent instructions in same basic block
16462 // Only equality constraints
16463 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16464 // Only one replacement instruction
16465 //
16466 // ---------EXAMPLE----------------------------------------------------------
16467 //
16468 // // pertinent parts of existing instructions in architecture description
16469 // instruct movI(iRegINoSp dst, iRegI src)
16470 // %{
16471 // match(Set dst (CopyI src));
16472 // %}
16473 //
16474 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16475 // %{
16476 // match(Set dst (AddI dst src));
16477 // effect(KILL cr);
16478 // %}
16479 //
16480 // // Change (inc mov) to lea
16481 // peephole %{
16482 // // increment preceded by register-register move
16483 // peepmatch ( incI_iReg movI );
16484 // // require that the destination register of the increment
16485 // // match the destination register of the move
16486 // peepconstraint ( 0.dst == 1.dst );
16487 // // construct a replacement instruction that sets
16488 // // the destination to ( move's source register + one )
16489 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16490 // %}
16491 //
16492
16493 // Implementation no longer uses movX instructions since
16494 // machine-independent system no longer uses CopyX nodes.
16495 //
16496 // peephole
16497 // %{
16498 // peepmatch (incI_iReg movI);
16499 // peepconstraint (0.dst == 1.dst);
16500 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16501 // %}
16502
16503 // peephole
16504 // %{
16505 // peepmatch (decI_iReg movI);
16506 // peepconstraint (0.dst == 1.dst);
16507 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16508 // %}
16509
16510 // peephole
16511 // %{
16512 // peepmatch (addI_iReg_imm movI);
16513 // peepconstraint (0.dst == 1.dst);
16514 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16515 // %}
16516
16517 // peephole
16518 // %{
16519 // peepmatch (incL_iReg movL);
16520 // peepconstraint (0.dst == 1.dst);
16521 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16522 // %}
16523
16524 // peephole
16525 // %{
16526 // peepmatch (decL_iReg movL);
16527 // peepconstraint (0.dst == 1.dst);
16528 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16529 // %}
16530
16531 // peephole
16532 // %{
16533 // peepmatch (addL_iReg_imm movL);
16534 // peepconstraint (0.dst == 1.dst);
16535 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16536 // %}
16537
16538 // peephole
16539 // %{
16540 // peepmatch (addP_iReg_imm movP);
16541 // peepconstraint (0.dst == 1.dst);
16542 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16543 // %}
16544
16545 // // Change load of spilled value to only a spill
16546 // instruct storeI(memory mem, iRegI src)
16547 // %{
16548 // match(Set mem (StoreI mem src));
16549 // %}
16550 //
16551 // instruct loadI(iRegINoSp dst, memory mem)
16552 // %{
16553 // match(Set dst (LoadI mem));
16554 // %}
16555 //
16556
16557 //----------SMARTSPILL RULES---------------------------------------------------
16558 // These must follow all instruction definitions as they use the names
16559 // defined in the instructions definitions.
16560
16561 // Local Variables:
16562 // mode: c++
16563 // End: