1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1716
1717 //=============================================================================
1718
1719 #ifndef PRODUCT
1720 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1721 st->print("nop \t# %d bytes pad for loops and calls", _count);
1722 }
1723 #endif
1724
1725 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1726 for (int i = 0; i < _count; i++) {
1727 __ nop();
1728 }
1729 }
1730
1731 uint MachNopNode::size(PhaseRegAlloc*) const {
1732 return _count * NativeInstruction::instruction_size;
1733 }
1734
1735 //=============================================================================
1736 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1737
1738 int ConstantTable::calculate_table_base_offset() const {
1739 return 0; // absolute addressing, no offset
1740 }
1741
1742 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1743 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1744 ShouldNotReachHere();
1745 }
1746
1747 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1748 // Empty encoding
1749 }
1750
1751 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1752 return 0;
1753 }
1754
1755 #ifndef PRODUCT
1756 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1757 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1758 }
1759 #endif
1760
1761 #ifndef PRODUCT
1762 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1763 Compile* C = ra_->C;
1764
1765 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1766
1767 if (C->output()->need_stack_bang(framesize))
1768 st->print("# stack bang size=%d\n\t", framesize);
1769
1770 if (VM_Version::use_rop_protection()) {
1771 st->print("ldr zr, [lr]\n\t");
1772 st->print("paciaz\n\t");
1773 }
1774 if (framesize < ((1 << 9) + 2 * wordSize)) {
1775 st->print("sub sp, sp, #%d\n\t", framesize);
1776 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1777 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1778 } else {
1779 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1780 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1781 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1782 st->print("sub sp, sp, rscratch1");
1783 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
1910 if (reg < slots_of_int_registers) {
1911 return rc_int;
1912 }
1913
1914 // we have 32 float register * 8 halves
1915 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1916 if (reg < slots_of_int_registers + slots_of_float_registers) {
1917 return rc_float;
1918 }
1919
1920 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1921 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1922 return rc_predicate;
1923 }
1924
1925 // Between predicate regs & stack is the flags.
1926 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1927
1928 return rc_stack;
1929 }
1930
1931 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1932 Compile* C = ra_->C;
1933
1934 // Get registers to move.
1935 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1936 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1937 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1938 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1939
1940 enum RC src_hi_rc = rc_class(src_hi);
1941 enum RC src_lo_rc = rc_class(src_lo);
1942 enum RC dst_hi_rc = rc_class(dst_hi);
1943 enum RC dst_lo_rc = rc_class(dst_lo);
1944
1945 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1946
1947 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1948 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1949 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1950 "expected aligned-adjacent pairs");
1951 }
1952
1953 if (src_lo == dst_lo && src_hi == dst_hi) {
1954 return 0; // Self copy, no move.
1955 }
1956
1957 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1958 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1959 int src_offset = ra_->reg2offset(src_lo);
1960 int dst_offset = ra_->reg2offset(dst_lo);
1961
1962 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1963 uint ireg = ideal_reg();
1964 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1965 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1966 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1967 if (ireg == Op_VecA && masm) {
1968 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1969 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1970 // stack->stack
1971 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1974 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1975 sve_vector_reg_size_in_bytes);
1976 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1977 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1978 sve_vector_reg_size_in_bytes);
1979 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1980 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1981 as_FloatRegister(Matcher::_regEncode[src_lo]),
1982 as_FloatRegister(Matcher::_regEncode[src_lo]));
1983 } else {
1984 ShouldNotReachHere();
1985 }
1986 } else if (masm) {
1987 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1988 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1989 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1990 // stack->stack
1991 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1992 if (ireg == Op_VecD) {
1993 __ unspill(rscratch1, true, src_offset);
1994 __ spill(rscratch1, true, dst_offset);
1995 } else {
1996 __ spill_copy128(src_offset, dst_offset);
1997 }
1998 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1999 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2000 ireg == Op_VecD ? __ T8B : __ T16B,
2001 as_FloatRegister(Matcher::_regEncode[src_lo]));
2002 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2003 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2004 ireg == Op_VecD ? __ D : __ Q,
2005 ra_->reg2offset(dst_lo));
2006 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2007 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2008 ireg == Op_VecD ? __ D : __ Q,
2009 ra_->reg2offset(src_lo));
2010 } else {
2011 ShouldNotReachHere();
2012 }
2013 }
2014 } else if (masm) {
2015 switch (src_lo_rc) {
2016 case rc_int:
2017 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2018 if (is64) {
2019 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2020 as_Register(Matcher::_regEncode[src_lo]));
2021 } else {
2022 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2023 as_Register(Matcher::_regEncode[src_lo]));
2024 }
2025 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2026 if (is64) {
2027 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2028 as_Register(Matcher::_regEncode[src_lo]));
2029 } else {
2030 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2031 as_Register(Matcher::_regEncode[src_lo]));
2032 }
2033 } else { // gpr --> stack spill
2034 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2035 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2036 }
2037 break;
2038 case rc_float:
2039 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2040 if (is64) {
2041 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2042 as_FloatRegister(Matcher::_regEncode[src_lo]));
2043 } else {
2044 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2045 as_FloatRegister(Matcher::_regEncode[src_lo]));
2046 }
2047 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2048 if (is64) {
2049 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 as_FloatRegister(Matcher::_regEncode[src_lo]));
2051 } else {
2052 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 }
2055 } else { // fpr --> stack spill
2056 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2057 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2058 is64 ? __ D : __ S, dst_offset);
2059 }
2060 break;
2061 case rc_stack:
2062 if (dst_lo_rc == rc_int) { // stack --> gpr load
2063 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2064 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2065 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2066 is64 ? __ D : __ S, src_offset);
2067 } else if (dst_lo_rc == rc_predicate) {
2068 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2069 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2070 } else { // stack --> stack copy
2071 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2072 if (ideal_reg() == Op_RegVectMask) {
2073 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2074 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2075 } else {
2076 __ unspill(rscratch1, is64, src_offset);
2077 __ spill(rscratch1, is64, dst_offset);
2078 }
2079 }
2080 break;
2081 case rc_predicate:
2082 if (dst_lo_rc == rc_predicate) {
2083 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2084 } else if (dst_lo_rc == rc_stack) {
2085 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2086 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2087 } else {
2088 assert(false, "bad src and dst rc_class combination.");
2089 ShouldNotReachHere();
2090 }
2091 break;
2092 default:
2093 assert(false, "bad rc_class for spill");
2094 ShouldNotReachHere();
2095 }
2096 }
2097
2098 if (st) {
2099 st->print("spill ");
2100 if (src_lo_rc == rc_stack) {
2101 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2102 } else {
2103 st->print("%s -> ", Matcher::regName[src_lo]);
2104 }
2105 if (dst_lo_rc == rc_stack) {
2106 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2107 } else {
2108 st->print("%s", Matcher::regName[dst_lo]);
2109 }
2110 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2111 int vsize = 0;
2112 switch (ideal_reg()) {
2113 case Op_VecD:
2114 vsize = 64;
2115 break;
2116 case Op_VecX:
2117 vsize = 128;
2118 break;
2119 case Op_VecA:
2120 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2121 break;
2122 default:
2123 assert(false, "bad register type for spill");
2124 ShouldNotReachHere();
2125 }
2126 st->print("\t# vector spill size = %d", vsize);
2127 } else if (ideal_reg() == Op_RegVectMask) {
2128 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2129 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2130 st->print("\t# predicate spill size = %d", vsize);
2131 } else {
2132 st->print("\t# spill size = %d", is64 ? 64 : 32);
2133 }
2134 }
2135
2136 return 0;
2137
2138 }
2139
2140 #ifndef PRODUCT
2141 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2142 if (!ra_)
2143 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2144 else
2145 implementation(nullptr, ra_, false, st);
2146 }
2147 #endif
2148
2149 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2150 implementation(masm, ra_, false, nullptr);
2151 }
2152
2153 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2154 return MachNode::size(ra_);
2155 }
2156
2157 //=============================================================================
2158
2159 #ifndef PRODUCT
2160 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2161 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2162 int reg = ra_->get_reg_first(this);
2163 st->print("add %s, rsp, #%d]\t# box lock",
2164 Matcher::regName[reg], offset);
2165 }
2166 #endif
2167
2168 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2169 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2170 int reg = ra_->get_encode(this);
2171
2172 // This add will handle any 24-bit signed offset. 24 bits allows an
2173 // 8 megabyte stack frame.
2174 __ add(as_Register(reg), sp, offset);
2175 }
2176
2177 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2178 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2179 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2180
2181 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2182 return NativeInstruction::instruction_size;
2183 } else {
2184 return 2 * NativeInstruction::instruction_size;
2185 }
2186 }
2187
2188 ///=============================================================================
2189 #ifndef PRODUCT
2190 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2191 {
2192 st->print_cr("# MachVEPNode");
2193 if (!_verified) {
2194 st->print_cr("\t load_class");
2195 } else {
2196 st->print_cr("\t unpack_inline_arg");
2197 }
2198 }
2199 #endif
2200
2201 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2202 {
2203 if (!_verified) {
2204 __ ic_check(1);
2205 } else {
2206 if (ra_->C->stub_function() == nullptr) {
2207 // Emit the entry barrier in a temporary frame before unpacking because
2208 // it can deopt, which would require packing the scalarized args again.
2209 __ verified_entry(ra_->C, 0);
2210 __ entry_barrier();
2211 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2212 __ remove_frame(framesize, false);
2213 }
2214 // Unpack inline type args passed as oop and then jump to
2215 // the verified entry point (skipping the unverified entry).
2216 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2217 // Emit code for verified entry and save increment for stack repair on return
2218 __ verified_entry(ra_->C, sp_inc);
2219 if (Compile::current()->output()->in_scratch_emit_size()) {
2220 Label dummy_verified_entry;
2221 __ b(dummy_verified_entry);
2222 } else {
2223 __ b(*_verified_entry);
2224 }
2225 }
2226 }
2227
2228 //=============================================================================
2229 #ifndef PRODUCT
2230 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2231 {
2232 st->print_cr("# MachUEPNode");
2233 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2234 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2235 st->print_cr("\tcmpw rscratch1, r10");
2236 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2237 }
2238 #endif
2239
2240 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2241 {
2242 __ ic_check(InteriorEntryAlignment);
2243 }
2244
2245 // REQUIRED EMIT CODE
2246
2247 //=============================================================================
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259
2260 int offset = __ offset();
2261 Label start;
2262 __ bind(start);
2263 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2264
2265 int entry_offset = __ offset();
2266 __ b(start);
2267
2268 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2269 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2270 "out of bounds read in post-call NOP check");
2271 __ end_a_stub();
2272 return entry_offset;
2273 }
2274
2275 // REQUIRED MATCHER CODE
2276
2277 //=============================================================================
2278
2279 bool Matcher::match_rule_supported(int opcode) {
2280 if (!has_match_rule(opcode))
2281 return false;
2282
2283 switch (opcode) {
2284 case Op_OnSpinWait:
2285 return VM_Version::supports_on_spin_wait();
2286 case Op_CacheWB:
2287 case Op_CacheWBPreSync:
2288 case Op_CacheWBPostSync:
2289 if (!VM_Version::supports_data_cache_line_flush()) {
2290 return false;
2291 }
2292 break;
2293 case Op_ExpandBits:
2294 case Op_CompressBits:
2295 if (!VM_Version::supports_svebitperm()) {
2296 return false;
2297 }
2298 break;
2299 case Op_FmaF:
2300 case Op_FmaD:
2301 case Op_FmaVF:
2302 case Op_FmaVD:
2303 if (!UseFMA) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaHF:
2308 // UseFMA flag also needs to be checked along with FEAT_FP16
2309 if (!UseFMA || !is_feat_fp16_supported()) {
2310 return false;
2311 }
2312 break;
2313 case Op_AddHF:
2314 case Op_SubHF:
2315 case Op_MulHF:
2316 case Op_DivHF:
2317 case Op_MinHF:
2318 case Op_MaxHF:
2319 case Op_SqrtHF:
2320 // Half-precision floating point scalar operations require FEAT_FP16
2321 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2322 // features are supported.
2323 if (!is_feat_fp16_supported()) {
2324 return false;
2325 }
2326 break;
2327 }
2328
2329 return true; // Per default match rules are supported.
2330 }
2331
2332 const RegMask* Matcher::predicate_reg_mask(void) {
2333 return &_PR_REG_mask;
2334 }
2335
2336 bool Matcher::supports_vector_calling_convention(void) {
2337 return EnableVectorSupport;
2338 }
2339
2340 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2341 assert(EnableVectorSupport, "sanity");
2342 int lo = V0_num;
2343 int hi = V0_H_num;
2344 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2345 hi = V0_K_num;
2346 }
2347 return OptoRegPair(hi, lo);
2348 }
2349
2350 // Is this branch offset short enough that a short branch can be used?
2351 //
2352 // NOTE: If the platform does not provide any short branch variants, then
2353 // this method should return false for offset 0.
2354 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2355 // The passed offset is relative to address of the branch.
2356
2357 return (-32768 <= offset && offset < 32768);
2358 }
2359
2360 // Vector width in bytes.
2361 int Matcher::vector_width_in_bytes(BasicType bt) {
2362 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2363 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2364 // Minimum 2 values in vector
2365 if (size < 2*type2aelembytes(bt)) size = 0;
2366 // But never < 4
2367 if (size < 4) size = 0;
2368 return size;
2369 }
2370
2371 // Limits on vector size (number of elements) loaded into vector.
2372 int Matcher::max_vector_size(const BasicType bt) {
2373 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2374 }
2375
2376 int Matcher::min_vector_size(const BasicType bt) {
2377 // Usually, the shortest vector length supported by AArch64 ISA and
2378 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2379 // vectors in a few special cases.
2380 int size;
2381 switch(bt) {
2382 case T_BOOLEAN:
2383 // Load/store a vector mask with only 2 elements for vector types
2384 // such as "2I/2F/2L/2D".
2385 size = 2;
2386 break;
2387 case T_BYTE:
2388 // Generate a "4B" vector, to support vector cast between "8B/16B"
2389 // and "4S/4I/4L/4F/4D".
2390 size = 4;
2391 break;
2392 case T_SHORT:
2393 // Generate a "2S" vector, to support vector cast between "4S/8S"
2394 // and "2I/2L/2F/2D".
2395 size = 2;
2396 break;
2397 default:
2398 // Limit the min vector length to 64-bit.
2399 size = 8 / type2aelembytes(bt);
2400 // The number of elements in a vector should be at least 2.
2401 size = MAX2(size, 2);
2402 }
2403
2404 int max_size = max_vector_size(bt);
2405 return MIN2(size, max_size);
2406 }
2407
2408 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2409 return Matcher::max_vector_size(bt);
2410 }
2411
2412 // Actual max scalable vector register length.
2413 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2414 return Matcher::max_vector_size(bt);
2415 }
2416
2417 // Vector ideal reg.
2418 uint Matcher::vector_ideal_reg(int len) {
2419 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2420 return Op_VecA;
2421 }
2422 switch(len) {
2423 // For 16-bit/32-bit mask vector, reuse VecD.
2424 case 2:
2425 case 4:
2426 case 8: return Op_VecD;
2427 case 16: return Op_VecX;
2428 }
2429 ShouldNotReachHere();
2430 return 0;
2431 }
2432
2433 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2434 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2435 switch (ideal_reg) {
2436 case Op_VecA: return new vecAOper();
2437 case Op_VecD: return new vecDOper();
2438 case Op_VecX: return new vecXOper();
2439 }
2440 ShouldNotReachHere();
2441 return nullptr;
2442 }
2443
2444 bool Matcher::is_reg2reg_move(MachNode* m) {
2445 return false;
2446 }
2447
2448 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2449 return false;
2450 }
2451
2452 bool Matcher::is_generic_vector(MachOper* opnd) {
2453 return opnd->opcode() == VREG;
2454 }
2455
2456 #ifdef ASSERT
2457 // Return whether or not this register is ever used as an argument.
2458 bool Matcher::can_be_java_arg(int reg)
2459 {
2460 return
2461 reg == R0_num || reg == R0_H_num ||
2462 reg == R1_num || reg == R1_H_num ||
2463 reg == R2_num || reg == R2_H_num ||
2464 reg == R3_num || reg == R3_H_num ||
2465 reg == R4_num || reg == R4_H_num ||
2466 reg == R5_num || reg == R5_H_num ||
2467 reg == R6_num || reg == R6_H_num ||
2468 reg == R7_num || reg == R7_H_num ||
2469 reg == V0_num || reg == V0_H_num ||
2470 reg == V1_num || reg == V1_H_num ||
2471 reg == V2_num || reg == V2_H_num ||
2472 reg == V3_num || reg == V3_H_num ||
2473 reg == V4_num || reg == V4_H_num ||
2474 reg == V5_num || reg == V5_H_num ||
2475 reg == V6_num || reg == V6_H_num ||
2476 reg == V7_num || reg == V7_H_num;
2477 }
2478 #endif
2479
2480 uint Matcher::int_pressure_limit()
2481 {
2482 // JDK-8183543: When taking the number of available registers as int
2483 // register pressure threshold, the jtreg test:
2484 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2485 // failed due to C2 compilation failure with
2486 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2487 //
2488 // A derived pointer is live at CallNode and then is flagged by RA
2489 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2490 // derived pointers and lastly fail to spill after reaching maximum
2491 // number of iterations. Lowering the default pressure threshold to
2492 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2493 // a high register pressure area of the code so that split_DEF can
2494 // generate DefinitionSpillCopy for the derived pointer.
2495 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2496 if (!PreserveFramePointer) {
2497 // When PreserveFramePointer is off, frame pointer is allocatable,
2498 // but different from other SOC registers, it is excluded from
2499 // fatproj's mask because its save type is No-Save. Decrease 1 to
2500 // ensure high pressure at fatproj when PreserveFramePointer is off.
2501 // See check_pressure_at_fatproj().
2502 default_int_pressure_threshold--;
2503 }
2504 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2505 }
2506
2507 uint Matcher::float_pressure_limit()
2508 {
2509 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2510 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2511 }
2512
2513 const RegMask& Matcher::divI_proj_mask() {
2514 ShouldNotReachHere();
2515 return RegMask::EMPTY;
2516 }
2517
2518 // Register for MODI projection of divmodI.
2519 const RegMask& Matcher::modI_proj_mask() {
2520 ShouldNotReachHere();
2521 return RegMask::EMPTY;
2522 }
2523
2524 // Register for DIVL projection of divmodL.
2525 const RegMask& Matcher::divL_proj_mask() {
2526 ShouldNotReachHere();
2527 return RegMask::EMPTY;
2528 }
2529
2530 // Register for MODL projection of divmodL.
2531 const RegMask& Matcher::modL_proj_mask() {
2532 ShouldNotReachHere();
2533 return RegMask::EMPTY;
2534 }
2535
2536 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2537 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2538 Node* u = addp->fast_out(i);
2539 if (u->is_LoadStore()) {
2540 // On AArch64, LoadStoreNodes (i.e. compare and swap
2541 // instructions) only take register indirect as an operand, so
2542 // any attempt to use an AddPNode as an input to a LoadStoreNode
2543 // must fail.
2544 return false;
2545 }
2546 if (u->is_Mem()) {
2547 int opsize = u->as_Mem()->memory_size();
2548 assert(opsize > 0, "unexpected memory operand size");
2549 if (u->as_Mem()->memory_size() != (1<<shift)) {
2550 return false;
2551 }
2552 }
2553 }
2554 return true;
2555 }
2556
2557 // Convert BoolTest condition to Assembler condition.
2558 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2559 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2560 Assembler::Condition result;
2561 switch(cond) {
2562 case BoolTest::eq:
2563 result = Assembler::EQ; break;
2564 case BoolTest::ne:
2565 result = Assembler::NE; break;
2566 case BoolTest::le:
2567 result = Assembler::LE; break;
2568 case BoolTest::ge:
2569 result = Assembler::GE; break;
2570 case BoolTest::lt:
2571 result = Assembler::LT; break;
2572 case BoolTest::gt:
2573 result = Assembler::GT; break;
2574 case BoolTest::ule:
2575 result = Assembler::LS; break;
2576 case BoolTest::uge:
2577 result = Assembler::HS; break;
2578 case BoolTest::ult:
2579 result = Assembler::LO; break;
2580 case BoolTest::ugt:
2581 result = Assembler::HI; break;
2582 case BoolTest::overflow:
2583 result = Assembler::VS; break;
2584 case BoolTest::no_overflow:
2585 result = Assembler::VC; break;
2586 default:
2587 ShouldNotReachHere();
2588 return Assembler::Condition(-1);
2589 }
2590
2591 // Check conversion
2592 if (cond & BoolTest::unsigned_compare) {
2593 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2594 } else {
2595 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2596 }
2597
2598 return result;
2599 }
2600
2601 // Binary src (Replicate con)
2602 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2603 if (n == nullptr || m == nullptr) {
2604 return false;
2605 }
2606
2607 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2608 return false;
2609 }
2610
2611 Node* imm_node = m->in(1);
2612 if (!imm_node->is_Con()) {
2613 return false;
2614 }
2615
2616 const Type* t = imm_node->bottom_type();
2617 if (!(t->isa_int() || t->isa_long())) {
2618 return false;
2619 }
2620
2621 switch (n->Opcode()) {
2622 case Op_AndV:
2623 case Op_OrV:
2624 case Op_XorV: {
2625 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2626 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2627 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2628 }
2629 case Op_AddVB:
2630 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2631 case Op_AddVS:
2632 case Op_AddVI:
2633 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2634 case Op_AddVL:
2635 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2636 default:
2637 return false;
2638 }
2639 }
2640
2641 // (XorV src (Replicate m1))
2642 // (XorVMask src (MaskAll m1))
2643 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2644 if (n != nullptr && m != nullptr) {
2645 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2646 VectorNode::is_all_ones_vector(m);
2647 }
2648 return false;
2649 }
2650
2651 // Should the matcher clone input 'm' of node 'n'?
2652 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2653 if (is_vshift_con_pattern(n, m) ||
2654 is_vector_bitwise_not_pattern(n, m) ||
2655 is_valid_sve_arith_imm_pattern(n, m) ||
2656 is_encode_and_store_pattern(n, m)) {
2657 mstack.push(m, Visit);
2658 return true;
2659 }
2660 return false;
2661 }
2662
2663 // Should the Matcher clone shifts on addressing modes, expecting them
2664 // to be subsumed into complex addressing expressions or compute them
2665 // into registers?
2666 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2667
2668 // Loads and stores with indirect memory input (e.g., volatile loads and
2669 // stores) do not subsume the input into complex addressing expressions. If
2670 // the addressing expression is input to at least one such load or store, do
2671 // not clone the addressing expression. Query needs_acquiring_load and
2672 // needs_releasing_store as a proxy for indirect memory input, as it is not
2673 // possible to directly query for indirect memory input at this stage.
2674 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2675 Node* n = m->fast_out(i);
2676 if (n->is_Load() && needs_acquiring_load(n)) {
2677 return false;
2678 }
2679 if (n->is_Store() && needs_releasing_store(n)) {
2680 return false;
2681 }
2682 }
2683
2684 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2685 return true;
2686 }
2687
2688 Node *off = m->in(AddPNode::Offset);
2689 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2690 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2691 // Are there other uses besides address expressions?
2692 !is_visited(off)) {
2693 address_visited.set(off->_idx); // Flag as address_visited
2694 mstack.push(off->in(2), Visit);
2695 Node *conv = off->in(1);
2696 if (conv->Opcode() == Op_ConvI2L &&
2697 // Are there other uses besides address expressions?
2698 !is_visited(conv)) {
2699 address_visited.set(conv->_idx); // Flag as address_visited
2700 mstack.push(conv->in(1), Pre_Visit);
2701 } else {
2702 mstack.push(conv, Pre_Visit);
2703 }
2704 address_visited.test_set(m->_idx); // Flag as address_visited
2705 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2706 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2707 return true;
2708 } else if (off->Opcode() == Op_ConvI2L &&
2709 // Are there other uses besides address expressions?
2710 !is_visited(off)) {
2711 address_visited.test_set(m->_idx); // Flag as address_visited
2712 address_visited.set(off->_idx); // Flag as address_visited
2713 mstack.push(off->in(1), Pre_Visit);
2714 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2715 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2716 return true;
2717 }
2718 return false;
2719 }
2720
2721 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2722 { \
2723 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2724 guarantee(DISP == 0, "mode not permitted for volatile"); \
2725 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2726 __ INSN(REG, as_Register(BASE)); \
2727 }
2728
2729
2730 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2731 {
2732 Address::extend scale;
2733
2734 // Hooboy, this is fugly. We need a way to communicate to the
2735 // encoder that the index needs to be sign extended, so we have to
2736 // enumerate all the cases.
2737 switch (opcode) {
2738 case INDINDEXSCALEDI2L:
2739 case INDINDEXSCALEDI2LN:
2740 case INDINDEXI2L:
2741 case INDINDEXI2LN:
2742 scale = Address::sxtw(size);
2743 break;
2744 default:
2745 scale = Address::lsl(size);
2746 }
2747
2748 if (index == -1) {
2749 return Address(base, disp);
2750 } else {
2751 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2752 return Address(base, as_Register(index), scale);
2753 }
2754 }
2755
2756
2757 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2758 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2759 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2760 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2761 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2762
2763 // Used for all non-volatile memory accesses. The use of
2764 // $mem->opcode() to discover whether this pattern uses sign-extended
2765 // offsets is something of a kludge.
2766 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2767 Register reg, int opcode,
2768 Register base, int index, int scale, int disp,
2769 int size_in_memory)
2770 {
2771 Address addr = mem2address(opcode, base, index, scale, disp);
2772 if (addr.getMode() == Address::base_plus_offset) {
2773 /* Fix up any out-of-range offsets. */
2774 assert_different_registers(rscratch1, base);
2775 assert_different_registers(rscratch1, reg);
2776 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2777 }
2778 (masm->*insn)(reg, addr);
2779 }
2780
2781 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2782 FloatRegister reg, int opcode,
2783 Register base, int index, int size, int disp,
2784 int size_in_memory)
2785 {
2786 Address::extend scale;
2787
2788 switch (opcode) {
2789 case INDINDEXSCALEDI2L:
2790 case INDINDEXSCALEDI2LN:
2791 scale = Address::sxtw(size);
2792 break;
2793 default:
2794 scale = Address::lsl(size);
2795 }
2796
2797 if (index == -1) {
2798 // Fix up any out-of-range offsets.
2799 assert_different_registers(rscratch1, base);
2800 Address addr = Address(base, disp);
2801 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2802 (masm->*insn)(reg, addr);
2803 } else {
2804 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2805 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2806 }
2807 }
2808
2809 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2810 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2811 int opcode, Register base, int index, int size, int disp)
2812 {
2813 if (index == -1) {
2814 (masm->*insn)(reg, T, Address(base, disp));
2815 } else {
2816 assert(disp == 0, "unsupported address mode");
2817 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2818 }
2819 }
2820
2821 %}
2822
2823
2824
2825 //----------ENCODING BLOCK-----------------------------------------------------
2826 // This block specifies the encoding classes used by the compiler to
2827 // output byte streams. Encoding classes are parameterized macros
2828 // used by Machine Instruction Nodes in order to generate the bit
2829 // encoding of the instruction. Operands specify their base encoding
2830 // interface with the interface keyword. There are currently
2831 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2832 // COND_INTER. REG_INTER causes an operand to generate a function
2833 // which returns its register number when queried. CONST_INTER causes
2834 // an operand to generate a function which returns the value of the
2835 // constant when queried. MEMORY_INTER causes an operand to generate
2836 // four functions which return the Base Register, the Index Register,
2837 // the Scale Value, and the Offset Value of the operand when queried.
2838 // COND_INTER causes an operand to generate six functions which return
2839 // the encoding code (ie - encoding bits for the instruction)
2840 // associated with each basic boolean condition for a conditional
2841 // instruction.
2842 //
2843 // Instructions specify two basic values for encoding. Again, a
2844 // function is available to check if the constant displacement is an
2845 // oop. They use the ins_encode keyword to specify their encoding
2846 // classes (which must be a sequence of enc_class names, and their
2847 // parameters, specified in the encoding block), and they use the
2848 // opcode keyword to specify, in order, their primary, secondary, and
2849 // tertiary opcode. Only the opcode sections which a particular
2850 // instruction needs for encoding need to be specified.
2851 encode %{
2852 // Build emit functions for each basic byte or larger field in the
2853 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2854 // from C++ code in the enc_class source block. Emit functions will
2855 // live in the main source block for now. In future, we can
2856 // generalize this by adding a syntax that specifies the sizes of
2857 // fields in an order, so that the adlc can build the emit functions
2858 // automagically
2859
2860 // catch all for unimplemented encodings
2861 enc_class enc_unimplemented %{
2862 __ unimplemented("C2 catch all");
2863 %}
2864
2865 // BEGIN Non-volatile memory access
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2966 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2974 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2982 Register src_reg = as_Register($src$$reg);
2983 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_strb0(memory1 mem) %{
2990 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2992 %}
2993
2994 // This encoding class is generated automatically from ad_encode.m4.
2995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2996 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2997 Register src_reg = as_Register($src$$reg);
2998 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3000 %}
3001
3002 // This encoding class is generated automatically from ad_encode.m4.
3003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3004 enc_class aarch64_enc_strh0(memory2 mem) %{
3005 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3006 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3007 %}
3008
3009 // This encoding class is generated automatically from ad_encode.m4.
3010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3011 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3012 Register src_reg = as_Register($src$$reg);
3013 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3014 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3015 %}
3016
3017 // This encoding class is generated automatically from ad_encode.m4.
3018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3019 enc_class aarch64_enc_strw0(memory4 mem) %{
3020 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3021 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3022 %}
3023
3024 // This encoding class is generated automatically from ad_encode.m4.
3025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3026 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3027 Register src_reg = as_Register($src$$reg);
3028 // we sometimes get asked to store the stack pointer into the
3029 // current thread -- we cannot do that directly on AArch64
3030 if (src_reg == r31_sp) {
3031 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3032 __ mov(rscratch2, sp);
3033 src_reg = rscratch2;
3034 }
3035 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3036 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3037 %}
3038
3039 // This encoding class is generated automatically from ad_encode.m4.
3040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3041 enc_class aarch64_enc_str0(memory8 mem) %{
3042 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3043 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3044 %}
3045
3046 // This encoding class is generated automatically from ad_encode.m4.
3047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3048 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3049 FloatRegister src_reg = as_FloatRegister($src$$reg);
3050 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3051 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3052 %}
3053
3054 // This encoding class is generated automatically from ad_encode.m4.
3055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3056 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3057 FloatRegister src_reg = as_FloatRegister($src$$reg);
3058 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3065 __ membar(Assembler::StoreStore);
3066 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3068 %}
3069
3070 // END Non-volatile memory access
3071
3072 // Vector loads and stores
3073 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3074 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3075 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3076 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3077 %}
3078
3079 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3080 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3081 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3082 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3083 %}
3084
3085 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3086 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3087 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3088 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3089 %}
3090
3091 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3092 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3093 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3094 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3095 %}
3096
3097 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3098 FloatRegister src_reg = as_FloatRegister($src$$reg);
3099 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3104 FloatRegister src_reg = as_FloatRegister($src$$reg);
3105 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3106 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3107 %}
3108
3109 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3110 FloatRegister src_reg = as_FloatRegister($src$$reg);
3111 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3112 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3113 %}
3114
3115 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3116 FloatRegister src_reg = as_FloatRegister($src$$reg);
3117 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3118 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3119 %}
3120
3121 // volatile loads and stores
3122
3123 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3124 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, stlrb);
3126 %}
3127
3128 enc_class aarch64_enc_stlrb0(memory mem) %{
3129 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3130 rscratch1, stlrb);
3131 %}
3132
3133 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3134 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3135 rscratch1, stlrh);
3136 %}
3137
3138 enc_class aarch64_enc_stlrh0(memory mem) %{
3139 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3140 rscratch1, stlrh);
3141 %}
3142
3143 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3144 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3145 rscratch1, stlrw);
3146 %}
3147
3148 enc_class aarch64_enc_stlrw0(memory mem) %{
3149 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3150 rscratch1, stlrw);
3151 %}
3152
3153 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3154 Register dst_reg = as_Register($dst$$reg);
3155 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, ldarb);
3157 __ sxtbw(dst_reg, dst_reg);
3158 %}
3159
3160 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3161 Register dst_reg = as_Register($dst$$reg);
3162 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, ldarb);
3164 __ sxtb(dst_reg, dst_reg);
3165 %}
3166
3167 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3168 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, ldarb);
3170 %}
3171
3172 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3173 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, ldarb);
3175 %}
3176
3177 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3178 Register dst_reg = as_Register($dst$$reg);
3179 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3180 rscratch1, ldarh);
3181 __ sxthw(dst_reg, dst_reg);
3182 %}
3183
3184 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3185 Register dst_reg = as_Register($dst$$reg);
3186 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3187 rscratch1, ldarh);
3188 __ sxth(dst_reg, dst_reg);
3189 %}
3190
3191 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3192 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3193 rscratch1, ldarh);
3194 %}
3195
3196 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3197 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarh);
3199 %}
3200
3201 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3202 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, ldarw);
3204 %}
3205
3206 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3207 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3208 rscratch1, ldarw);
3209 %}
3210
3211 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3212 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3213 rscratch1, ldar);
3214 %}
3215
3216 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3217 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, ldarw);
3219 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3220 %}
3221
3222 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3223 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, ldar);
3225 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3226 %}
3227
3228 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3229 Register src_reg = as_Register($src$$reg);
3230 // we sometimes get asked to store the stack pointer into the
3231 // current thread -- we cannot do that directly on AArch64
3232 if (src_reg == r31_sp) {
3233 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3234 __ mov(rscratch2, sp);
3235 src_reg = rscratch2;
3236 }
3237 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, stlr);
3239 %}
3240
3241 enc_class aarch64_enc_stlr0(memory mem) %{
3242 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3243 rscratch1, stlr);
3244 %}
3245
3246 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3247 {
3248 FloatRegister src_reg = as_FloatRegister($src$$reg);
3249 __ fmovs(rscratch2, src_reg);
3250 }
3251 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3252 rscratch1, stlrw);
3253 %}
3254
3255 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3256 {
3257 FloatRegister src_reg = as_FloatRegister($src$$reg);
3258 __ fmovd(rscratch2, src_reg);
3259 }
3260 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3261 rscratch1, stlr);
3262 %}
3263
3264 // synchronized read/update encodings
3265
3266 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3267 Register dst_reg = as_Register($dst$$reg);
3268 Register base = as_Register($mem$$base);
3269 int index = $mem$$index;
3270 int scale = $mem$$scale;
3271 int disp = $mem$$disp;
3272 if (index == -1) {
3273 if (disp != 0) {
3274 __ lea(rscratch1, Address(base, disp));
3275 __ ldaxr(dst_reg, rscratch1);
3276 } else {
3277 // TODO
3278 // should we ever get anything other than this case?
3279 __ ldaxr(dst_reg, base);
3280 }
3281 } else {
3282 Register index_reg = as_Register(index);
3283 if (disp == 0) {
3284 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3285 __ ldaxr(dst_reg, rscratch1);
3286 } else {
3287 __ lea(rscratch1, Address(base, disp));
3288 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3289 __ ldaxr(dst_reg, rscratch1);
3290 }
3291 }
3292 %}
3293
3294 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3295 Register src_reg = as_Register($src$$reg);
3296 Register base = as_Register($mem$$base);
3297 int index = $mem$$index;
3298 int scale = $mem$$scale;
3299 int disp = $mem$$disp;
3300 if (index == -1) {
3301 if (disp != 0) {
3302 __ lea(rscratch2, Address(base, disp));
3303 __ stlxr(rscratch1, src_reg, rscratch2);
3304 } else {
3305 // TODO
3306 // should we ever get anything other than this case?
3307 __ stlxr(rscratch1, src_reg, base);
3308 }
3309 } else {
3310 Register index_reg = as_Register(index);
3311 if (disp == 0) {
3312 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3313 __ stlxr(rscratch1, src_reg, rscratch2);
3314 } else {
3315 __ lea(rscratch2, Address(base, disp));
3316 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3317 __ stlxr(rscratch1, src_reg, rscratch2);
3318 }
3319 }
3320 __ cmpw(rscratch1, zr);
3321 %}
3322
3323 // prefetch encodings
3324
3325 enc_class aarch64_enc_prefetchw(memory mem) %{
3326 Register base = as_Register($mem$$base);
3327 int index = $mem$$index;
3328 int scale = $mem$$scale;
3329 int disp = $mem$$disp;
3330 if (index == -1) {
3331 // Fix up any out-of-range offsets.
3332 assert_different_registers(rscratch1, base);
3333 Address addr = Address(base, disp);
3334 addr = __ legitimize_address(addr, 8, rscratch1);
3335 __ prfm(addr, PSTL1KEEP);
3336 } else {
3337 Register index_reg = as_Register(index);
3338 if (disp == 0) {
3339 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3340 } else {
3341 __ lea(rscratch1, Address(base, disp));
3342 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3343 }
3344 }
3345 %}
3346
3347 // mov encodings
3348
3349 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3350 uint32_t con = (uint32_t)$src$$constant;
3351 Register dst_reg = as_Register($dst$$reg);
3352 if (con == 0) {
3353 __ movw(dst_reg, zr);
3354 } else {
3355 __ movw(dst_reg, con);
3356 }
3357 %}
3358
3359 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3360 Register dst_reg = as_Register($dst$$reg);
3361 uint64_t con = (uint64_t)$src$$constant;
3362 if (con == 0) {
3363 __ mov(dst_reg, zr);
3364 } else {
3365 __ mov(dst_reg, con);
3366 }
3367 %}
3368
3369 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3370 Register dst_reg = as_Register($dst$$reg);
3371 address con = (address)$src$$constant;
3372 if (con == nullptr || con == (address)1) {
3373 ShouldNotReachHere();
3374 } else {
3375 relocInfo::relocType rtype = $src->constant_reloc();
3376 if (rtype == relocInfo::oop_type) {
3377 __ movoop(dst_reg, (jobject)con);
3378 } else if (rtype == relocInfo::metadata_type) {
3379 __ mov_metadata(dst_reg, (Metadata*)con);
3380 } else {
3381 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3382 // load fake address constants using a normal move
3383 if (! __ is_valid_AArch64_address(con) ||
3384 con < (address)(uintptr_t)os::vm_page_size()) {
3385 __ mov(dst_reg, con);
3386 } else {
3387 // no reloc so just use adrp and add
3388 uint64_t offset;
3389 __ adrp(dst_reg, con, offset);
3390 __ add(dst_reg, dst_reg, offset);
3391 }
3392 }
3393 }
3394 %}
3395
3396 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3397 Register dst_reg = as_Register($dst$$reg);
3398 __ mov(dst_reg, zr);
3399 %}
3400
3401 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3402 Register dst_reg = as_Register($dst$$reg);
3403 __ mov(dst_reg, (uint64_t)1);
3404 %}
3405
3406 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3407 Register dst_reg = as_Register($dst$$reg);
3408 address con = (address)$src$$constant;
3409 if (con == nullptr) {
3410 ShouldNotReachHere();
3411 } else {
3412 relocInfo::relocType rtype = $src->constant_reloc();
3413 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3414 __ set_narrow_oop(dst_reg, (jobject)con);
3415 }
3416 %}
3417
3418 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3419 Register dst_reg = as_Register($dst$$reg);
3420 __ mov(dst_reg, zr);
3421 %}
3422
3423 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3424 Register dst_reg = as_Register($dst$$reg);
3425 address con = (address)$src$$constant;
3426 if (con == nullptr) {
3427 ShouldNotReachHere();
3428 } else {
3429 relocInfo::relocType rtype = $src->constant_reloc();
3430 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3431 __ set_narrow_klass(dst_reg, (Klass *)con);
3432 }
3433 %}
3434
3435 // arithmetic encodings
3436
3437 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3438 Register dst_reg = as_Register($dst$$reg);
3439 Register src_reg = as_Register($src1$$reg);
3440 int32_t con = (int32_t)$src2$$constant;
3441 // add has primary == 0, subtract has primary == 1
3442 if ($primary) { con = -con; }
3443 if (con < 0) {
3444 __ subw(dst_reg, src_reg, -con);
3445 } else {
3446 __ addw(dst_reg, src_reg, con);
3447 }
3448 %}
3449
3450 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3451 Register dst_reg = as_Register($dst$$reg);
3452 Register src_reg = as_Register($src1$$reg);
3453 int32_t con = (int32_t)$src2$$constant;
3454 // add has primary == 0, subtract has primary == 1
3455 if ($primary) { con = -con; }
3456 if (con < 0) {
3457 __ sub(dst_reg, src_reg, -con);
3458 } else {
3459 __ add(dst_reg, src_reg, con);
3460 }
3461 %}
3462
3463 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3464 Register dst_reg = as_Register($dst$$reg);
3465 Register src1_reg = as_Register($src1$$reg);
3466 Register src2_reg = as_Register($src2$$reg);
3467 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3468 %}
3469
3470 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3471 Register dst_reg = as_Register($dst$$reg);
3472 Register src1_reg = as_Register($src1$$reg);
3473 Register src2_reg = as_Register($src2$$reg);
3474 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3475 %}
3476
3477 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3478 Register dst_reg = as_Register($dst$$reg);
3479 Register src1_reg = as_Register($src1$$reg);
3480 Register src2_reg = as_Register($src2$$reg);
3481 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3482 %}
3483
3484 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3485 Register dst_reg = as_Register($dst$$reg);
3486 Register src1_reg = as_Register($src1$$reg);
3487 Register src2_reg = as_Register($src2$$reg);
3488 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3489 %}
3490
3491 // compare instruction encodings
3492
3493 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3494 Register reg1 = as_Register($src1$$reg);
3495 Register reg2 = as_Register($src2$$reg);
3496 __ cmpw(reg1, reg2);
3497 %}
3498
3499 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3500 Register reg = as_Register($src1$$reg);
3501 int32_t val = $src2$$constant;
3502 if (val >= 0) {
3503 __ subsw(zr, reg, val);
3504 } else {
3505 __ addsw(zr, reg, -val);
3506 }
3507 %}
3508
3509 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3510 Register reg1 = as_Register($src1$$reg);
3511 uint32_t val = (uint32_t)$src2$$constant;
3512 __ movw(rscratch1, val);
3513 __ cmpw(reg1, rscratch1);
3514 %}
3515
3516 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3517 Register reg1 = as_Register($src1$$reg);
3518 Register reg2 = as_Register($src2$$reg);
3519 __ cmp(reg1, reg2);
3520 %}
3521
3522 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3523 Register reg = as_Register($src1$$reg);
3524 int64_t val = $src2$$constant;
3525 if (val >= 0) {
3526 __ subs(zr, reg, val);
3527 } else if (val != -val) {
3528 __ adds(zr, reg, -val);
3529 } else {
3530 // aargh, Long.MIN_VALUE is a special case
3531 __ orr(rscratch1, zr, (uint64_t)val);
3532 __ subs(zr, reg, rscratch1);
3533 }
3534 %}
3535
3536 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3537 Register reg1 = as_Register($src1$$reg);
3538 uint64_t val = (uint64_t)$src2$$constant;
3539 __ mov(rscratch1, val);
3540 __ cmp(reg1, rscratch1);
3541 %}
3542
3543 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3544 Register reg1 = as_Register($src1$$reg);
3545 Register reg2 = as_Register($src2$$reg);
3546 __ cmp(reg1, reg2);
3547 %}
3548
3549 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3550 Register reg1 = as_Register($src1$$reg);
3551 Register reg2 = as_Register($src2$$reg);
3552 __ cmpw(reg1, reg2);
3553 %}
3554
3555 enc_class aarch64_enc_testp(iRegP src) %{
3556 Register reg = as_Register($src$$reg);
3557 __ cmp(reg, zr);
3558 %}
3559
3560 enc_class aarch64_enc_testn(iRegN src) %{
3561 Register reg = as_Register($src$$reg);
3562 __ cmpw(reg, zr);
3563 %}
3564
3565 enc_class aarch64_enc_b(label lbl) %{
3566 Label *L = $lbl$$label;
3567 __ b(*L);
3568 %}
3569
3570 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3571 Label *L = $lbl$$label;
3572 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3573 %}
3574
3575 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3576 Label *L = $lbl$$label;
3577 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3578 %}
3579
3580 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3581 %{
3582 Register sub_reg = as_Register($sub$$reg);
3583 Register super_reg = as_Register($super$$reg);
3584 Register temp_reg = as_Register($temp$$reg);
3585 Register result_reg = as_Register($result$$reg);
3586
3587 Label miss;
3588 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3589 nullptr, &miss,
3590 /*set_cond_codes:*/ true);
3591 if ($primary) {
3592 __ mov(result_reg, zr);
3593 }
3594 __ bind(miss);
3595 %}
3596
3597 enc_class aarch64_enc_java_static_call(method meth) %{
3598 address addr = (address)$meth$$method;
3599 address call;
3600 if (!_method) {
3601 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3602 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3603 if (call == nullptr) {
3604 ciEnv::current()->record_failure("CodeCache is full");
3605 return;
3606 }
3607 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3608 // The NOP here is purely to ensure that eliding a call to
3609 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3610 __ nop();
3611 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3612 } else {
3613 int method_index = resolved_method_index(masm);
3614 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3615 : static_call_Relocation::spec(method_index);
3616 call = __ trampoline_call(Address(addr, rspec));
3617 if (call == nullptr) {
3618 ciEnv::current()->record_failure("CodeCache is full");
3619 return;
3620 }
3621 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3622 // Calls of the same statically bound method can share
3623 // a stub to the interpreter.
3624 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3625 } else {
3626 // Emit stub for static call
3627 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3628 if (stub == nullptr) {
3629 ciEnv::current()->record_failure("CodeCache is full");
3630 return;
3631 }
3632 }
3633 }
3634
3635 __ post_call_nop();
3636
3637 // Only non uncommon_trap calls need to reinitialize ptrue.
3638 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3639 __ reinitialize_ptrue();
3640 }
3641 %}
3642
3643 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3644 int method_index = resolved_method_index(masm);
3645 address call = __ ic_call((address)$meth$$method, method_index);
3646 if (call == nullptr) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 __ post_call_nop();
3651 if (Compile::current()->max_vector_size() > 0) {
3652 __ reinitialize_ptrue();
3653 }
3654 %}
3655
3656 enc_class aarch64_enc_call_epilog() %{
3657 if (VerifyStackAtCalls) {
3658 // Check that stack depth is unchanged: find majik cookie on stack
3659 __ call_Unimplemented();
3660 }
3661 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3662 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3663 // Search for the corresponding projection, get the register and emit code that initialized it.
3664 uint con = (tf()->range_cc()->cnt() - 1);
3665 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3666 ProjNode* proj = fast_out(i)->as_Proj();
3667 if (proj->_con == con) {
3668 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3669 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3670 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3671 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3672 __ cmp(r0, zr);
3673 __ cset(toReg, Assembler::NE);
3674 if (reg->is_stack()) {
3675 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3676 __ str(toReg, Address(sp, st_off));
3677 }
3678 break;
3679 }
3680 }
3681 if (return_value_is_used()) {
3682 // An inline type is returned as fields in multiple registers.
3683 // R0 either contains an oop if the inline type is buffered or a pointer
3684 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3685 // if the lowest bit is set to allow C2 to use the oop after null checking.
3686 // r0 &= (r0 & 1) - 1
3687 __ andr(rscratch1, r0, 0x1);
3688 __ sub(rscratch1, rscratch1, 0x1);
3689 __ andr(r0, r0, rscratch1);
3690 }
3691 }
3692 %}
3693
3694 enc_class aarch64_enc_java_to_runtime(method meth) %{
3695 // some calls to generated routines (arraycopy code) are scheduled
3696 // by C2 as runtime calls. if so we can call them using a br (they
3697 // will be in a reachable segment) otherwise we have to use a blr
3698 // which loads the absolute address into a register.
3699 address entry = (address)$meth$$method;
3700 CodeBlob *cb = CodeCache::find_blob(entry);
3701 if (cb) {
3702 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3703 if (call == nullptr) {
3704 ciEnv::current()->record_failure("CodeCache is full");
3705 return;
3706 }
3707 __ post_call_nop();
3708 } else {
3709 Label retaddr;
3710 // Make the anchor frame walkable
3711 __ adr(rscratch2, retaddr);
3712 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3713 __ lea(rscratch1, RuntimeAddress(entry));
3714 __ blr(rscratch1);
3715 __ bind(retaddr);
3716 __ post_call_nop();
3717 }
3718 if (Compile::current()->max_vector_size() > 0) {
3719 __ reinitialize_ptrue();
3720 }
3721 %}
3722
3723 enc_class aarch64_enc_rethrow() %{
3724 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3725 %}
3726
3727 enc_class aarch64_enc_ret() %{
3728 #ifdef ASSERT
3729 if (Compile::current()->max_vector_size() > 0) {
3730 __ verify_ptrue();
3731 }
3732 #endif
3733 __ ret(lr);
3734 %}
3735
3736 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3737 Register target_reg = as_Register($jump_target$$reg);
3738 __ br(target_reg);
3739 %}
3740
3741 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3742 Register target_reg = as_Register($jump_target$$reg);
3743 // exception oop should be in r0
3744 // ret addr has been popped into lr
3745 // callee expects it in r3
3746 __ mov(r3, lr);
3747 __ br(target_reg);
3748 %}
3749
3750 %}
3751
3752 //----------FRAME--------------------------------------------------------------
3753 // Definition of frame structure and management information.
3754 //
3755 // S T A C K L A Y O U T Allocators stack-slot number
3756 // | (to get allocators register number
3757 // G Owned by | | v add OptoReg::stack0())
3758 // r CALLER | |
3759 // o | +--------+ pad to even-align allocators stack-slot
3760 // w V | pad0 | numbers; owned by CALLER
3761 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3762 // h ^ | in | 5
3763 // | | args | 4 Holes in incoming args owned by SELF
3764 // | | | | 3
3765 // | | +--------+
3766 // V | | old out| Empty on Intel, window on Sparc
3767 // | old |preserve| Must be even aligned.
3768 // | SP-+--------+----> Matcher::_old_SP, even aligned
3769 // | | in | 3 area for Intel ret address
3770 // Owned by |preserve| Empty on Sparc.
3771 // SELF +--------+
3772 // | | pad2 | 2 pad to align old SP
3773 // | +--------+ 1
3774 // | | locks | 0
3775 // | +--------+----> OptoReg::stack0(), even aligned
3776 // | | pad1 | 11 pad to align new SP
3777 // | +--------+
3778 // | | | 10
3779 // | | spills | 9 spills
3780 // V | | 8 (pad0 slot for callee)
3781 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3782 // ^ | out | 7
3783 // | | args | 6 Holes in outgoing args owned by CALLEE
3784 // Owned by +--------+
3785 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3786 // | new |preserve| Must be even-aligned.
3787 // | SP-+--------+----> Matcher::_new_SP, even aligned
3788 // | | |
3789 //
3790 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3791 // known from SELF's arguments and the Java calling convention.
3792 // Region 6-7 is determined per call site.
3793 // Note 2: If the calling convention leaves holes in the incoming argument
3794 // area, those holes are owned by SELF. Holes in the outgoing area
3795 // are owned by the CALLEE. Holes should not be necessary in the
3796 // incoming area, as the Java calling convention is completely under
3797 // the control of the AD file. Doubles can be sorted and packed to
3798 // avoid holes. Holes in the outgoing arguments may be necessary for
3799 // varargs C calling conventions.
3800 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3801 // even aligned with pad0 as needed.
3802 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3803 // (the latter is true on Intel but is it false on AArch64?)
3804 // region 6-11 is even aligned; it may be padded out more so that
3805 // the region from SP to FP meets the minimum stack alignment.
3806 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3807 // alignment. Region 11, pad1, may be dynamically extended so that
3808 // SP meets the minimum alignment.
3809
3810 frame %{
3811 // These three registers define part of the calling convention
3812 // between compiled code and the interpreter.
3813
3814 // Inline Cache Register or Method for I2C.
3815 inline_cache_reg(R12);
3816
3817 // Number of stack slots consumed by locking an object
3818 sync_stack_slots(2);
3819
3820 // Compiled code's Frame Pointer
3821 frame_pointer(R31);
3822
3823 // Stack alignment requirement
3824 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3825
3826 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3827 // for calls to C. Supports the var-args backing area for register parms.
3828 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3829
3830 // The after-PROLOG location of the return address. Location of
3831 // return address specifies a type (REG or STACK) and a number
3832 // representing the register number (i.e. - use a register name) or
3833 // stack slot.
3834 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3835 // Otherwise, it is above the locks and verification slot and alignment word
3836 // TODO this may well be correct but need to check why that - 2 is there
3837 // ppc port uses 0 but we definitely need to allow for fixed_slots
3838 // which folds in the space used for monitors
3839 return_addr(STACK - 2 +
3840 align_up((Compile::current()->in_preserve_stack_slots() +
3841 Compile::current()->fixed_slots()),
3842 stack_alignment_in_slots()));
3843
3844 // Location of compiled Java return values. Same as C for now.
3845 return_value
3846 %{
3847 // TODO do we allow ideal_reg == Op_RegN???
3848 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3849 "only return normal values");
3850
3851 static const int lo[Op_RegL + 1] = { // enum name
3852 0, // Op_Node
3853 0, // Op_Set
3854 R0_num, // Op_RegN
3855 R0_num, // Op_RegI
3856 R0_num, // Op_RegP
3857 V0_num, // Op_RegF
3858 V0_num, // Op_RegD
3859 R0_num // Op_RegL
3860 };
3861
3862 static const int hi[Op_RegL + 1] = { // enum name
3863 0, // Op_Node
3864 0, // Op_Set
3865 OptoReg::Bad, // Op_RegN
3866 OptoReg::Bad, // Op_RegI
3867 R0_H_num, // Op_RegP
3868 OptoReg::Bad, // Op_RegF
3869 V0_H_num, // Op_RegD
3870 R0_H_num // Op_RegL
3871 };
3872
3873 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3874 %}
3875 %}
3876
3877 //----------ATTRIBUTES---------------------------------------------------------
3878 //----------Operand Attributes-------------------------------------------------
3879 op_attrib op_cost(1); // Required cost attribute
3880
3881 //----------Instruction Attributes---------------------------------------------
3882 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3883 ins_attrib ins_size(32); // Required size attribute (in bits)
3884 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3885 // a non-matching short branch variant
3886 // of some long branch?
3887 ins_attrib ins_alignment(4); // Required alignment attribute (must
3888 // be a power of 2) specifies the
3889 // alignment that some part of the
3890 // instruction (not necessarily the
3891 // start) requires. If > 1, a
3892 // compute_padding() function must be
3893 // provided for the instruction
3894
3895 // Whether this node is expanded during code emission into a sequence of
3896 // instructions and the first instruction can perform an implicit null check.
3897 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3898
3899 //----------OPERANDS-----------------------------------------------------------
3900 // Operand definitions must precede instruction definitions for correct parsing
3901 // in the ADLC because operands constitute user defined types which are used in
3902 // instruction definitions.
3903
3904 //----------Simple Operands----------------------------------------------------
3905
3906 // Integer operands 32 bit
3907 // 32 bit immediate
3908 operand immI()
3909 %{
3910 match(ConI);
3911
3912 op_cost(0);
3913 format %{ %}
3914 interface(CONST_INTER);
3915 %}
3916
3917 // 32 bit zero
3918 operand immI0()
3919 %{
3920 predicate(n->get_int() == 0);
3921 match(ConI);
3922
3923 op_cost(0);
3924 format %{ %}
3925 interface(CONST_INTER);
3926 %}
3927
3928 // 32 bit unit increment
3929 operand immI_1()
3930 %{
3931 predicate(n->get_int() == 1);
3932 match(ConI);
3933
3934 op_cost(0);
3935 format %{ %}
3936 interface(CONST_INTER);
3937 %}
3938
3939 // 32 bit unit decrement
3940 operand immI_M1()
3941 %{
3942 predicate(n->get_int() == -1);
3943 match(ConI);
3944
3945 op_cost(0);
3946 format %{ %}
3947 interface(CONST_INTER);
3948 %}
3949
3950 // Shift values for add/sub extension shift
3951 operand immIExt()
3952 %{
3953 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3954 match(ConI);
3955
3956 op_cost(0);
3957 format %{ %}
3958 interface(CONST_INTER);
3959 %}
3960
3961 operand immI_gt_1()
3962 %{
3963 predicate(n->get_int() > 1);
3964 match(ConI);
3965
3966 op_cost(0);
3967 format %{ %}
3968 interface(CONST_INTER);
3969 %}
3970
3971 operand immI_le_4()
3972 %{
3973 predicate(n->get_int() <= 4);
3974 match(ConI);
3975
3976 op_cost(0);
3977 format %{ %}
3978 interface(CONST_INTER);
3979 %}
3980
3981 operand immI_4()
3982 %{
3983 predicate(n->get_int() == 4);
3984 match(ConI);
3985
3986 op_cost(0);
3987 format %{ %}
3988 interface(CONST_INTER);
3989 %}
3990
3991 operand immI_16()
3992 %{
3993 predicate(n->get_int() == 16);
3994 match(ConI);
3995
3996 op_cost(0);
3997 format %{ %}
3998 interface(CONST_INTER);
3999 %}
4000
4001 operand immI_24()
4002 %{
4003 predicate(n->get_int() == 24);
4004 match(ConI);
4005
4006 op_cost(0);
4007 format %{ %}
4008 interface(CONST_INTER);
4009 %}
4010
4011 operand immI_32()
4012 %{
4013 predicate(n->get_int() == 32);
4014 match(ConI);
4015
4016 op_cost(0);
4017 format %{ %}
4018 interface(CONST_INTER);
4019 %}
4020
4021 operand immI_48()
4022 %{
4023 predicate(n->get_int() == 48);
4024 match(ConI);
4025
4026 op_cost(0);
4027 format %{ %}
4028 interface(CONST_INTER);
4029 %}
4030
4031 operand immI_56()
4032 %{
4033 predicate(n->get_int() == 56);
4034 match(ConI);
4035
4036 op_cost(0);
4037 format %{ %}
4038 interface(CONST_INTER);
4039 %}
4040
4041 operand immI_255()
4042 %{
4043 predicate(n->get_int() == 255);
4044 match(ConI);
4045
4046 op_cost(0);
4047 format %{ %}
4048 interface(CONST_INTER);
4049 %}
4050
4051 operand immI_65535()
4052 %{
4053 predicate(n->get_int() == 65535);
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 operand immI_positive()
4062 %{
4063 predicate(n->get_int() > 0);
4064 match(ConI);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 // BoolTest condition for signed compare
4072 operand immI_cmp_cond()
4073 %{
4074 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4075 match(ConI);
4076
4077 op_cost(0);
4078 format %{ %}
4079 interface(CONST_INTER);
4080 %}
4081
4082 // BoolTest condition for unsigned compare
4083 operand immI_cmpU_cond()
4084 %{
4085 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4086 match(ConI);
4087
4088 op_cost(0);
4089 format %{ %}
4090 interface(CONST_INTER);
4091 %}
4092
4093 operand immL_255()
4094 %{
4095 predicate(n->get_long() == 255L);
4096 match(ConL);
4097
4098 op_cost(0);
4099 format %{ %}
4100 interface(CONST_INTER);
4101 %}
4102
4103 operand immL_65535()
4104 %{
4105 predicate(n->get_long() == 65535L);
4106 match(ConL);
4107
4108 op_cost(0);
4109 format %{ %}
4110 interface(CONST_INTER);
4111 %}
4112
4113 operand immL_4294967295()
4114 %{
4115 predicate(n->get_long() == 4294967295L);
4116 match(ConL);
4117
4118 op_cost(0);
4119 format %{ %}
4120 interface(CONST_INTER);
4121 %}
4122
4123 operand immL_bitmask()
4124 %{
4125 predicate((n->get_long() != 0)
4126 && ((n->get_long() & 0xc000000000000000l) == 0)
4127 && is_power_of_2(n->get_long() + 1));
4128 match(ConL);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 operand immI_bitmask()
4136 %{
4137 predicate((n->get_int() != 0)
4138 && ((n->get_int() & 0xc0000000) == 0)
4139 && is_power_of_2(n->get_int() + 1));
4140 match(ConI);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 operand immL_positive_bitmaskI()
4148 %{
4149 predicate((n->get_long() != 0)
4150 && ((julong)n->get_long() < 0x80000000ULL)
4151 && is_power_of_2(n->get_long() + 1));
4152 match(ConL);
4153
4154 op_cost(0);
4155 format %{ %}
4156 interface(CONST_INTER);
4157 %}
4158
4159 // Scale values for scaled offset addressing modes (up to long but not quad)
4160 operand immIScale()
4161 %{
4162 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4163 match(ConI);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 // 5 bit signed integer
4171 operand immI5()
4172 %{
4173 predicate(Assembler::is_simm(n->get_int(), 5));
4174 match(ConI);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 // 7 bit unsigned integer
4182 operand immIU7()
4183 %{
4184 predicate(Assembler::is_uimm(n->get_int(), 7));
4185 match(ConI);
4186
4187 op_cost(0);
4188 format %{ %}
4189 interface(CONST_INTER);
4190 %}
4191
4192 // Offset for scaled or unscaled immediate loads and stores
4193 operand immIOffset()
4194 %{
4195 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4196 match(ConI);
4197
4198 op_cost(0);
4199 format %{ %}
4200 interface(CONST_INTER);
4201 %}
4202
4203 operand immIOffset1()
4204 %{
4205 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4206 match(ConI);
4207
4208 op_cost(0);
4209 format %{ %}
4210 interface(CONST_INTER);
4211 %}
4212
4213 operand immIOffset2()
4214 %{
4215 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4216 match(ConI);
4217
4218 op_cost(0);
4219 format %{ %}
4220 interface(CONST_INTER);
4221 %}
4222
4223 operand immIOffset4()
4224 %{
4225 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4226 match(ConI);
4227
4228 op_cost(0);
4229 format %{ %}
4230 interface(CONST_INTER);
4231 %}
4232
4233 operand immIOffset8()
4234 %{
4235 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4236 match(ConI);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 operand immIOffset16()
4244 %{
4245 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4246 match(ConI);
4247
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 operand immLOffset()
4254 %{
4255 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4256 match(ConL);
4257
4258 op_cost(0);
4259 format %{ %}
4260 interface(CONST_INTER);
4261 %}
4262
4263 operand immLoffset1()
4264 %{
4265 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4266 match(ConL);
4267
4268 op_cost(0);
4269 format %{ %}
4270 interface(CONST_INTER);
4271 %}
4272
4273 operand immLoffset2()
4274 %{
4275 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4276 match(ConL);
4277
4278 op_cost(0);
4279 format %{ %}
4280 interface(CONST_INTER);
4281 %}
4282
4283 operand immLoffset4()
4284 %{
4285 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4286 match(ConL);
4287
4288 op_cost(0);
4289 format %{ %}
4290 interface(CONST_INTER);
4291 %}
4292
4293 operand immLoffset8()
4294 %{
4295 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4296 match(ConL);
4297
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 operand immLoffset16()
4304 %{
4305 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4306 match(ConL);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 // 5 bit signed long integer
4314 operand immL5()
4315 %{
4316 predicate(Assembler::is_simm(n->get_long(), 5));
4317 match(ConL);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 7 bit unsigned long integer
4325 operand immLU7()
4326 %{
4327 predicate(Assembler::is_uimm(n->get_long(), 7));
4328 match(ConL);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 8 bit signed value.
4336 operand immI8()
4337 %{
4338 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4339 match(ConI);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit signed value (simm8), or #simm8 LSL 8.
4347 operand immIDupV()
4348 %{
4349 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4350 match(ConI);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 // 8 bit signed value (simm8), or #simm8 LSL 8.
4358 operand immLDupV()
4359 %{
4360 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4361 match(ConL);
4362
4363 op_cost(0);
4364 format %{ %}
4365 interface(CONST_INTER);
4366 %}
4367
4368 // 8 bit signed value (simm8), or #simm8 LSL 8.
4369 operand immHDupV()
4370 %{
4371 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4372 match(ConH);
4373
4374 op_cost(0);
4375 format %{ %}
4376 interface(CONST_INTER);
4377 %}
4378
4379 // 8 bit integer valid for vector add sub immediate
4380 operand immBAddSubV()
4381 %{
4382 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4383 match(ConI);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 // 32 bit integer valid for add sub immediate
4391 operand immIAddSub()
4392 %{
4393 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4394 match(ConI);
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 // 32 bit integer valid for vector add sub immediate
4401 operand immIAddSubV()
4402 %{
4403 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // 32 bit unsigned integer valid for logical immediate
4412
4413 operand immBLog()
4414 %{
4415 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4416 match(ConI);
4417
4418 op_cost(0);
4419 format %{ %}
4420 interface(CONST_INTER);
4421 %}
4422
4423 operand immSLog()
4424 %{
4425 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4426 match(ConI);
4427
4428 op_cost(0);
4429 format %{ %}
4430 interface(CONST_INTER);
4431 %}
4432
4433 operand immILog()
4434 %{
4435 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4436 match(ConI);
4437
4438 op_cost(0);
4439 format %{ %}
4440 interface(CONST_INTER);
4441 %}
4442
4443 // Integer operands 64 bit
4444 // 64 bit immediate
4445 operand immL()
4446 %{
4447 match(ConL);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // 64 bit zero
4455 operand immL0()
4456 %{
4457 predicate(n->get_long() == 0);
4458 match(ConL);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 // 64 bit unit decrement
4466 operand immL_M1()
4467 %{
4468 predicate(n->get_long() == -1);
4469 match(ConL);
4470
4471 op_cost(0);
4472 format %{ %}
4473 interface(CONST_INTER);
4474 %}
4475
4476 // 64 bit integer valid for add sub immediate
4477 operand immLAddSub()
4478 %{
4479 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4480 match(ConL);
4481 op_cost(0);
4482 format %{ %}
4483 interface(CONST_INTER);
4484 %}
4485
4486 // 64 bit integer valid for addv subv immediate
4487 operand immLAddSubV()
4488 %{
4489 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4490 match(ConL);
4491
4492 op_cost(0);
4493 format %{ %}
4494 interface(CONST_INTER);
4495 %}
4496
4497 // 64 bit integer valid for logical immediate
4498 operand immLLog()
4499 %{
4500 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4501 match(ConL);
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // Long Immediate: low 32-bit mask
4508 operand immL_32bits()
4509 %{
4510 predicate(n->get_long() == 0xFFFFFFFFL);
4511 match(ConL);
4512 op_cost(0);
4513 format %{ %}
4514 interface(CONST_INTER);
4515 %}
4516
4517 // Pointer operands
4518 // Pointer Immediate
4519 operand immP()
4520 %{
4521 match(ConP);
4522
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // nullptr Pointer Immediate
4529 operand immP0()
4530 %{
4531 predicate(n->get_ptr() == 0);
4532 match(ConP);
4533
4534 op_cost(0);
4535 format %{ %}
4536 interface(CONST_INTER);
4537 %}
4538
4539 // Pointer Immediate One
4540 // this is used in object initialization (initial object header)
4541 operand immP_1()
4542 %{
4543 predicate(n->get_ptr() == 1);
4544 match(ConP);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 // AOT Runtime Constants Address
4552 operand immAOTRuntimeConstantsAddress()
4553 %{
4554 // Check if the address is in the range of AOT Runtime Constants
4555 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4556 match(ConP);
4557
4558 op_cost(0);
4559 format %{ %}
4560 interface(CONST_INTER);
4561 %}
4562
4563 // Float and Double operands
4564 // Double Immediate
4565 operand immD()
4566 %{
4567 match(ConD);
4568 op_cost(0);
4569 format %{ %}
4570 interface(CONST_INTER);
4571 %}
4572
4573 // Double Immediate: +0.0d
4574 operand immD0()
4575 %{
4576 predicate(jlong_cast(n->getd()) == 0);
4577 match(ConD);
4578
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 // constant 'double +0.0'.
4585 operand immDPacked()
4586 %{
4587 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4588 match(ConD);
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 // Float Immediate
4595 operand immF()
4596 %{
4597 match(ConF);
4598 op_cost(0);
4599 format %{ %}
4600 interface(CONST_INTER);
4601 %}
4602
4603 // Float Immediate: +0.0f.
4604 operand immF0()
4605 %{
4606 predicate(jint_cast(n->getf()) == 0);
4607 match(ConF);
4608
4609 op_cost(0);
4610 format %{ %}
4611 interface(CONST_INTER);
4612 %}
4613
4614 // Half Float (FP16) Immediate
4615 operand immH()
4616 %{
4617 match(ConH);
4618 op_cost(0);
4619 format %{ %}
4620 interface(CONST_INTER);
4621 %}
4622
4623 //
4624 operand immFPacked()
4625 %{
4626 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4627 match(ConF);
4628 op_cost(0);
4629 format %{ %}
4630 interface(CONST_INTER);
4631 %}
4632
4633 // Narrow pointer operands
4634 // Narrow Pointer Immediate
4635 operand immN()
4636 %{
4637 match(ConN);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 // Narrow nullptr Pointer Immediate
4645 operand immN0()
4646 %{
4647 predicate(n->get_narrowcon() == 0);
4648 match(ConN);
4649
4650 op_cost(0);
4651 format %{ %}
4652 interface(CONST_INTER);
4653 %}
4654
4655 operand immNKlass()
4656 %{
4657 match(ConNKlass);
4658
4659 op_cost(0);
4660 format %{ %}
4661 interface(CONST_INTER);
4662 %}
4663
4664 // Integer 32 bit Register Operands
4665 // Integer 32 bitRegister (excludes SP)
4666 operand iRegI()
4667 %{
4668 constraint(ALLOC_IN_RC(any_reg32));
4669 match(RegI);
4670 match(iRegINoSp);
4671 op_cost(0);
4672 format %{ %}
4673 interface(REG_INTER);
4674 %}
4675
4676 // Integer 32 bit Register not Special
4677 operand iRegINoSp()
4678 %{
4679 constraint(ALLOC_IN_RC(no_special_reg32));
4680 match(RegI);
4681 op_cost(0);
4682 format %{ %}
4683 interface(REG_INTER);
4684 %}
4685
4686 // Integer 64 bit Register Operands
4687 // Integer 64 bit Register (includes SP)
4688 operand iRegL()
4689 %{
4690 constraint(ALLOC_IN_RC(any_reg));
4691 match(RegL);
4692 match(iRegLNoSp);
4693 op_cost(0);
4694 format %{ %}
4695 interface(REG_INTER);
4696 %}
4697
4698 // Integer 64 bit Register not Special
4699 operand iRegLNoSp()
4700 %{
4701 constraint(ALLOC_IN_RC(no_special_reg));
4702 match(RegL);
4703 match(iRegL_R0);
4704 format %{ %}
4705 interface(REG_INTER);
4706 %}
4707
4708 // Pointer Register Operands
4709 // Pointer Register
4710 operand iRegP()
4711 %{
4712 constraint(ALLOC_IN_RC(ptr_reg));
4713 match(RegP);
4714 match(iRegPNoSp);
4715 match(iRegP_R0);
4716 //match(iRegP_R2);
4717 //match(iRegP_R4);
4718 match(iRegP_R5);
4719 match(thread_RegP);
4720 op_cost(0);
4721 format %{ %}
4722 interface(REG_INTER);
4723 %}
4724
4725 // Pointer 64 bit Register not Special
4726 operand iRegPNoSp()
4727 %{
4728 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4729 match(RegP);
4730 // match(iRegP);
4731 // match(iRegP_R0);
4732 // match(iRegP_R2);
4733 // match(iRegP_R4);
4734 // match(iRegP_R5);
4735 // match(thread_RegP);
4736 op_cost(0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // This operand is not allowed to use rfp even if
4742 // rfp is not used to hold the frame pointer.
4743 operand iRegPNoSpNoRfp()
4744 %{
4745 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4746 match(RegP);
4747 match(iRegPNoSp);
4748 op_cost(0);
4749 format %{ %}
4750 interface(REG_INTER);
4751 %}
4752
4753 // Pointer 64 bit Register R0 only
4754 operand iRegP_R0()
4755 %{
4756 constraint(ALLOC_IN_RC(r0_reg));
4757 match(RegP);
4758 // match(iRegP);
4759 match(iRegPNoSp);
4760 op_cost(0);
4761 format %{ %}
4762 interface(REG_INTER);
4763 %}
4764
4765 // Pointer 64 bit Register R1 only
4766 operand iRegP_R1()
4767 %{
4768 constraint(ALLOC_IN_RC(r1_reg));
4769 match(RegP);
4770 // match(iRegP);
4771 match(iRegPNoSp);
4772 op_cost(0);
4773 format %{ %}
4774 interface(REG_INTER);
4775 %}
4776
4777 // Pointer 64 bit Register R2 only
4778 operand iRegP_R2()
4779 %{
4780 constraint(ALLOC_IN_RC(r2_reg));
4781 match(RegP);
4782 // match(iRegP);
4783 match(iRegPNoSp);
4784 op_cost(0);
4785 format %{ %}
4786 interface(REG_INTER);
4787 %}
4788
4789 // Pointer 64 bit Register R3 only
4790 operand iRegP_R3()
4791 %{
4792 constraint(ALLOC_IN_RC(r3_reg));
4793 match(RegP);
4794 // match(iRegP);
4795 match(iRegPNoSp);
4796 op_cost(0);
4797 format %{ %}
4798 interface(REG_INTER);
4799 %}
4800
4801 // Pointer 64 bit Register R4 only
4802 operand iRegP_R4()
4803 %{
4804 constraint(ALLOC_IN_RC(r4_reg));
4805 match(RegP);
4806 // match(iRegP);
4807 match(iRegPNoSp);
4808 op_cost(0);
4809 format %{ %}
4810 interface(REG_INTER);
4811 %}
4812
4813 // Pointer 64 bit Register R5 only
4814 operand iRegP_R5()
4815 %{
4816 constraint(ALLOC_IN_RC(r5_reg));
4817 match(RegP);
4818 // match(iRegP);
4819 match(iRegPNoSp);
4820 op_cost(0);
4821 format %{ %}
4822 interface(REG_INTER);
4823 %}
4824
4825 // Pointer 64 bit Register R10 only
4826 operand iRegP_R10()
4827 %{
4828 constraint(ALLOC_IN_RC(r10_reg));
4829 match(RegP);
4830 // match(iRegP);
4831 match(iRegPNoSp);
4832 op_cost(0);
4833 format %{ %}
4834 interface(REG_INTER);
4835 %}
4836
4837 // Long 64 bit Register R0 only
4838 operand iRegL_R0()
4839 %{
4840 constraint(ALLOC_IN_RC(r0_reg));
4841 match(RegL);
4842 match(iRegLNoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Long 64 bit Register R11 only
4849 operand iRegL_R11()
4850 %{
4851 constraint(ALLOC_IN_RC(r11_reg));
4852 match(RegL);
4853 match(iRegLNoSp);
4854 op_cost(0);
4855 format %{ %}
4856 interface(REG_INTER);
4857 %}
4858
4859 // Register R0 only
4860 operand iRegI_R0()
4861 %{
4862 constraint(ALLOC_IN_RC(int_r0_reg));
4863 match(RegI);
4864 match(iRegINoSp);
4865 op_cost(0);
4866 format %{ %}
4867 interface(REG_INTER);
4868 %}
4869
4870 // Register R2 only
4871 operand iRegI_R2()
4872 %{
4873 constraint(ALLOC_IN_RC(int_r2_reg));
4874 match(RegI);
4875 match(iRegINoSp);
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881 // Register R3 only
4882 operand iRegI_R3()
4883 %{
4884 constraint(ALLOC_IN_RC(int_r3_reg));
4885 match(RegI);
4886 match(iRegINoSp);
4887 op_cost(0);
4888 format %{ %}
4889 interface(REG_INTER);
4890 %}
4891
4892
4893 // Register R4 only
4894 operand iRegI_R4()
4895 %{
4896 constraint(ALLOC_IN_RC(int_r4_reg));
4897 match(RegI);
4898 match(iRegINoSp);
4899 op_cost(0);
4900 format %{ %}
4901 interface(REG_INTER);
4902 %}
4903
4904
4905 // Pointer Register Operands
4906 // Narrow Pointer Register
4907 operand iRegN()
4908 %{
4909 constraint(ALLOC_IN_RC(any_reg32));
4910 match(RegN);
4911 match(iRegNNoSp);
4912 op_cost(0);
4913 format %{ %}
4914 interface(REG_INTER);
4915 %}
4916
4917 // Integer 64 bit Register not Special
4918 operand iRegNNoSp()
4919 %{
4920 constraint(ALLOC_IN_RC(no_special_reg32));
4921 match(RegN);
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 // Float Register
4928 // Float register operands
4929 operand vRegF()
4930 %{
4931 constraint(ALLOC_IN_RC(float_reg));
4932 match(RegF);
4933
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939 // Double Register
4940 // Double register operands
4941 operand vRegD()
4942 %{
4943 constraint(ALLOC_IN_RC(double_reg));
4944 match(RegD);
4945
4946 op_cost(0);
4947 format %{ %}
4948 interface(REG_INTER);
4949 %}
4950
4951 // Generic vector class. This will be used for
4952 // all vector operands, including NEON and SVE.
4953 operand vReg()
4954 %{
4955 constraint(ALLOC_IN_RC(dynamic));
4956 match(VecA);
4957 match(VecD);
4958 match(VecX);
4959
4960 op_cost(0);
4961 format %{ %}
4962 interface(REG_INTER);
4963 %}
4964
4965 operand vReg_V10()
4966 %{
4967 constraint(ALLOC_IN_RC(v10_veca_reg));
4968 match(vReg);
4969
4970 op_cost(0);
4971 format %{ %}
4972 interface(REG_INTER);
4973 %}
4974
4975 operand vReg_V11()
4976 %{
4977 constraint(ALLOC_IN_RC(v11_veca_reg));
4978 match(vReg);
4979
4980 op_cost(0);
4981 format %{ %}
4982 interface(REG_INTER);
4983 %}
4984
4985 operand vReg_V12()
4986 %{
4987 constraint(ALLOC_IN_RC(v12_veca_reg));
4988 match(vReg);
4989
4990 op_cost(0);
4991 format %{ %}
4992 interface(REG_INTER);
4993 %}
4994
4995 operand vReg_V13()
4996 %{
4997 constraint(ALLOC_IN_RC(v13_veca_reg));
4998 match(vReg);
4999
5000 op_cost(0);
5001 format %{ %}
5002 interface(REG_INTER);
5003 %}
5004
5005 operand vReg_V17()
5006 %{
5007 constraint(ALLOC_IN_RC(v17_veca_reg));
5008 match(vReg);
5009
5010 op_cost(0);
5011 format %{ %}
5012 interface(REG_INTER);
5013 %}
5014
5015 operand vReg_V18()
5016 %{
5017 constraint(ALLOC_IN_RC(v18_veca_reg));
5018 match(vReg);
5019
5020 op_cost(0);
5021 format %{ %}
5022 interface(REG_INTER);
5023 %}
5024
5025 operand vReg_V23()
5026 %{
5027 constraint(ALLOC_IN_RC(v23_veca_reg));
5028 match(vReg);
5029
5030 op_cost(0);
5031 format %{ %}
5032 interface(REG_INTER);
5033 %}
5034
5035 operand vReg_V24()
5036 %{
5037 constraint(ALLOC_IN_RC(v24_veca_reg));
5038 match(vReg);
5039
5040 op_cost(0);
5041 format %{ %}
5042 interface(REG_INTER);
5043 %}
5044
5045 operand vecA()
5046 %{
5047 constraint(ALLOC_IN_RC(vectora_reg));
5048 match(VecA);
5049
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 operand vecD()
5056 %{
5057 constraint(ALLOC_IN_RC(vectord_reg));
5058 match(VecD);
5059
5060 op_cost(0);
5061 format %{ %}
5062 interface(REG_INTER);
5063 %}
5064
5065 operand vecX()
5066 %{
5067 constraint(ALLOC_IN_RC(vectorx_reg));
5068 match(VecX);
5069
5070 op_cost(0);
5071 format %{ %}
5072 interface(REG_INTER);
5073 %}
5074
5075 operand vRegD_V0()
5076 %{
5077 constraint(ALLOC_IN_RC(v0_reg));
5078 match(RegD);
5079 op_cost(0);
5080 format %{ %}
5081 interface(REG_INTER);
5082 %}
5083
5084 operand vRegD_V1()
5085 %{
5086 constraint(ALLOC_IN_RC(v1_reg));
5087 match(RegD);
5088 op_cost(0);
5089 format %{ %}
5090 interface(REG_INTER);
5091 %}
5092
5093 operand vRegD_V2()
5094 %{
5095 constraint(ALLOC_IN_RC(v2_reg));
5096 match(RegD);
5097 op_cost(0);
5098 format %{ %}
5099 interface(REG_INTER);
5100 %}
5101
5102 operand vRegD_V3()
5103 %{
5104 constraint(ALLOC_IN_RC(v3_reg));
5105 match(RegD);
5106 op_cost(0);
5107 format %{ %}
5108 interface(REG_INTER);
5109 %}
5110
5111 operand vRegD_V4()
5112 %{
5113 constraint(ALLOC_IN_RC(v4_reg));
5114 match(RegD);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 operand vRegD_V5()
5121 %{
5122 constraint(ALLOC_IN_RC(v5_reg));
5123 match(RegD);
5124 op_cost(0);
5125 format %{ %}
5126 interface(REG_INTER);
5127 %}
5128
5129 operand vRegD_V6()
5130 %{
5131 constraint(ALLOC_IN_RC(v6_reg));
5132 match(RegD);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 operand vRegD_V7()
5139 %{
5140 constraint(ALLOC_IN_RC(v7_reg));
5141 match(RegD);
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 operand vRegD_V12()
5148 %{
5149 constraint(ALLOC_IN_RC(v12_reg));
5150 match(RegD);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 operand vRegD_V13()
5157 %{
5158 constraint(ALLOC_IN_RC(v13_reg));
5159 match(RegD);
5160 op_cost(0);
5161 format %{ %}
5162 interface(REG_INTER);
5163 %}
5164
5165 operand pReg()
5166 %{
5167 constraint(ALLOC_IN_RC(pr_reg));
5168 match(RegVectMask);
5169 match(pRegGov);
5170 op_cost(0);
5171 format %{ %}
5172 interface(REG_INTER);
5173 %}
5174
5175 operand pRegGov()
5176 %{
5177 constraint(ALLOC_IN_RC(gov_pr));
5178 match(RegVectMask);
5179 match(pReg);
5180 op_cost(0);
5181 format %{ %}
5182 interface(REG_INTER);
5183 %}
5184
5185 operand pRegGov_P0()
5186 %{
5187 constraint(ALLOC_IN_RC(p0_reg));
5188 match(RegVectMask);
5189 op_cost(0);
5190 format %{ %}
5191 interface(REG_INTER);
5192 %}
5193
5194 operand pRegGov_P1()
5195 %{
5196 constraint(ALLOC_IN_RC(p1_reg));
5197 match(RegVectMask);
5198 op_cost(0);
5199 format %{ %}
5200 interface(REG_INTER);
5201 %}
5202
5203 // Flags register, used as output of signed compare instructions
5204
5205 // note that on AArch64 we also use this register as the output for
5206 // for floating point compare instructions (CmpF CmpD). this ensures
5207 // that ordered inequality tests use GT, GE, LT or LE none of which
5208 // pass through cases where the result is unordered i.e. one or both
5209 // inputs to the compare is a NaN. this means that the ideal code can
5210 // replace e.g. a GT with an LE and not end up capturing the NaN case
5211 // (where the comparison should always fail). EQ and NE tests are
5212 // always generated in ideal code so that unordered folds into the NE
5213 // case, matching the behaviour of AArch64 NE.
5214 //
5215 // This differs from x86 where the outputs of FP compares use a
5216 // special FP flags registers and where compares based on this
5217 // register are distinguished into ordered inequalities (cmpOpUCF) and
5218 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5219 // to explicitly handle the unordered case in branches. x86 also has
5220 // to include extra CMoveX rules to accept a cmpOpUCF input.
5221
5222 operand rFlagsReg()
5223 %{
5224 constraint(ALLOC_IN_RC(int_flags));
5225 match(RegFlags);
5226
5227 op_cost(0);
5228 format %{ "RFLAGS" %}
5229 interface(REG_INTER);
5230 %}
5231
5232 // Flags register, used as output of unsigned compare instructions
5233 operand rFlagsRegU()
5234 %{
5235 constraint(ALLOC_IN_RC(int_flags));
5236 match(RegFlags);
5237
5238 op_cost(0);
5239 format %{ "RFLAGSU" %}
5240 interface(REG_INTER);
5241 %}
5242
5243 // Special Registers
5244
5245 // Method Register
5246 operand inline_cache_RegP(iRegP reg)
5247 %{
5248 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5249 match(reg);
5250 match(iRegPNoSp);
5251 op_cost(0);
5252 format %{ %}
5253 interface(REG_INTER);
5254 %}
5255
5256 // Thread Register
5257 operand thread_RegP(iRegP reg)
5258 %{
5259 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5260 match(reg);
5261 op_cost(0);
5262 format %{ %}
5263 interface(REG_INTER);
5264 %}
5265
5266 //----------Memory Operands----------------------------------------------------
5267
5268 operand indirect(iRegP reg)
5269 %{
5270 constraint(ALLOC_IN_RC(ptr_reg));
5271 match(reg);
5272 op_cost(0);
5273 format %{ "[$reg]" %}
5274 interface(MEMORY_INTER) %{
5275 base($reg);
5276 index(0xffffffff);
5277 scale(0x0);
5278 disp(0x0);
5279 %}
5280 %}
5281
5282 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5283 %{
5284 constraint(ALLOC_IN_RC(ptr_reg));
5285 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5286 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5287 op_cost(0);
5288 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5289 interface(MEMORY_INTER) %{
5290 base($reg);
5291 index($ireg);
5292 scale($scale);
5293 disp(0x0);
5294 %}
5295 %}
5296
5297 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5298 %{
5299 constraint(ALLOC_IN_RC(ptr_reg));
5300 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5301 match(AddP reg (LShiftL lreg scale));
5302 op_cost(0);
5303 format %{ "$reg, $lreg lsl($scale)" %}
5304 interface(MEMORY_INTER) %{
5305 base($reg);
5306 index($lreg);
5307 scale($scale);
5308 disp(0x0);
5309 %}
5310 %}
5311
5312 operand indIndexI2L(iRegP reg, iRegI ireg)
5313 %{
5314 constraint(ALLOC_IN_RC(ptr_reg));
5315 match(AddP reg (ConvI2L ireg));
5316 op_cost(0);
5317 format %{ "$reg, $ireg, 0, I2L" %}
5318 interface(MEMORY_INTER) %{
5319 base($reg);
5320 index($ireg);
5321 scale(0x0);
5322 disp(0x0);
5323 %}
5324 %}
5325
5326 operand indIndex(iRegP reg, iRegL lreg)
5327 %{
5328 constraint(ALLOC_IN_RC(ptr_reg));
5329 match(AddP reg lreg);
5330 op_cost(0);
5331 format %{ "$reg, $lreg" %}
5332 interface(MEMORY_INTER) %{
5333 base($reg);
5334 index($lreg);
5335 scale(0x0);
5336 disp(0x0);
5337 %}
5338 %}
5339
5340 operand indOffI1(iRegP reg, immIOffset1 off)
5341 %{
5342 constraint(ALLOC_IN_RC(ptr_reg));
5343 match(AddP reg off);
5344 op_cost(0);
5345 format %{ "[$reg, $off]" %}
5346 interface(MEMORY_INTER) %{
5347 base($reg);
5348 index(0xffffffff);
5349 scale(0x0);
5350 disp($off);
5351 %}
5352 %}
5353
5354 operand indOffI2(iRegP reg, immIOffset2 off)
5355 %{
5356 constraint(ALLOC_IN_RC(ptr_reg));
5357 match(AddP reg off);
5358 op_cost(0);
5359 format %{ "[$reg, $off]" %}
5360 interface(MEMORY_INTER) %{
5361 base($reg);
5362 index(0xffffffff);
5363 scale(0x0);
5364 disp($off);
5365 %}
5366 %}
5367
5368 operand indOffI4(iRegP reg, immIOffset4 off)
5369 %{
5370 constraint(ALLOC_IN_RC(ptr_reg));
5371 match(AddP reg off);
5372 op_cost(0);
5373 format %{ "[$reg, $off]" %}
5374 interface(MEMORY_INTER) %{
5375 base($reg);
5376 index(0xffffffff);
5377 scale(0x0);
5378 disp($off);
5379 %}
5380 %}
5381
5382 operand indOffI8(iRegP reg, immIOffset8 off)
5383 %{
5384 constraint(ALLOC_IN_RC(ptr_reg));
5385 match(AddP reg off);
5386 op_cost(0);
5387 format %{ "[$reg, $off]" %}
5388 interface(MEMORY_INTER) %{
5389 base($reg);
5390 index(0xffffffff);
5391 scale(0x0);
5392 disp($off);
5393 %}
5394 %}
5395
5396 operand indOffI16(iRegP reg, immIOffset16 off)
5397 %{
5398 constraint(ALLOC_IN_RC(ptr_reg));
5399 match(AddP reg off);
5400 op_cost(0);
5401 format %{ "[$reg, $off]" %}
5402 interface(MEMORY_INTER) %{
5403 base($reg);
5404 index(0xffffffff);
5405 scale(0x0);
5406 disp($off);
5407 %}
5408 %}
5409
5410 operand indOffL1(iRegP reg, immLoffset1 off)
5411 %{
5412 constraint(ALLOC_IN_RC(ptr_reg));
5413 match(AddP reg off);
5414 op_cost(0);
5415 format %{ "[$reg, $off]" %}
5416 interface(MEMORY_INTER) %{
5417 base($reg);
5418 index(0xffffffff);
5419 scale(0x0);
5420 disp($off);
5421 %}
5422 %}
5423
5424 operand indOffL2(iRegP reg, immLoffset2 off)
5425 %{
5426 constraint(ALLOC_IN_RC(ptr_reg));
5427 match(AddP reg off);
5428 op_cost(0);
5429 format %{ "[$reg, $off]" %}
5430 interface(MEMORY_INTER) %{
5431 base($reg);
5432 index(0xffffffff);
5433 scale(0x0);
5434 disp($off);
5435 %}
5436 %}
5437
5438 operand indOffL4(iRegP reg, immLoffset4 off)
5439 %{
5440 constraint(ALLOC_IN_RC(ptr_reg));
5441 match(AddP reg off);
5442 op_cost(0);
5443 format %{ "[$reg, $off]" %}
5444 interface(MEMORY_INTER) %{
5445 base($reg);
5446 index(0xffffffff);
5447 scale(0x0);
5448 disp($off);
5449 %}
5450 %}
5451
5452 operand indOffL8(iRegP reg, immLoffset8 off)
5453 %{
5454 constraint(ALLOC_IN_RC(ptr_reg));
5455 match(AddP reg off);
5456 op_cost(0);
5457 format %{ "[$reg, $off]" %}
5458 interface(MEMORY_INTER) %{
5459 base($reg);
5460 index(0xffffffff);
5461 scale(0x0);
5462 disp($off);
5463 %}
5464 %}
5465
5466 operand indOffL16(iRegP reg, immLoffset16 off)
5467 %{
5468 constraint(ALLOC_IN_RC(ptr_reg));
5469 match(AddP reg off);
5470 op_cost(0);
5471 format %{ "[$reg, $off]" %}
5472 interface(MEMORY_INTER) %{
5473 base($reg);
5474 index(0xffffffff);
5475 scale(0x0);
5476 disp($off);
5477 %}
5478 %}
5479
5480 operand indirectX2P(iRegL reg)
5481 %{
5482 constraint(ALLOC_IN_RC(ptr_reg));
5483 match(CastX2P reg);
5484 op_cost(0);
5485 format %{ "[$reg]\t# long -> ptr" %}
5486 interface(MEMORY_INTER) %{
5487 base($reg);
5488 index(0xffffffff);
5489 scale(0x0);
5490 disp(0x0);
5491 %}
5492 %}
5493
5494 operand indOffX2P(iRegL reg, immLOffset off)
5495 %{
5496 constraint(ALLOC_IN_RC(ptr_reg));
5497 match(AddP (CastX2P reg) off);
5498 op_cost(0);
5499 format %{ "[$reg, $off]\t# long -> ptr" %}
5500 interface(MEMORY_INTER) %{
5501 base($reg);
5502 index(0xffffffff);
5503 scale(0x0);
5504 disp($off);
5505 %}
5506 %}
5507
5508 operand indirectN(iRegN reg)
5509 %{
5510 predicate(CompressedOops::shift() == 0);
5511 constraint(ALLOC_IN_RC(ptr_reg));
5512 match(DecodeN reg);
5513 op_cost(0);
5514 format %{ "[$reg]\t# narrow" %}
5515 interface(MEMORY_INTER) %{
5516 base($reg);
5517 index(0xffffffff);
5518 scale(0x0);
5519 disp(0x0);
5520 %}
5521 %}
5522
5523 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5524 %{
5525 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5526 constraint(ALLOC_IN_RC(ptr_reg));
5527 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5528 op_cost(0);
5529 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5530 interface(MEMORY_INTER) %{
5531 base($reg);
5532 index($ireg);
5533 scale($scale);
5534 disp(0x0);
5535 %}
5536 %}
5537
5538 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5539 %{
5540 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5541 constraint(ALLOC_IN_RC(ptr_reg));
5542 match(AddP (DecodeN reg) (LShiftL lreg scale));
5543 op_cost(0);
5544 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5545 interface(MEMORY_INTER) %{
5546 base($reg);
5547 index($lreg);
5548 scale($scale);
5549 disp(0x0);
5550 %}
5551 %}
5552
5553 operand indIndexI2LN(iRegN reg, iRegI ireg)
5554 %{
5555 predicate(CompressedOops::shift() == 0);
5556 constraint(ALLOC_IN_RC(ptr_reg));
5557 match(AddP (DecodeN reg) (ConvI2L ireg));
5558 op_cost(0);
5559 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5560 interface(MEMORY_INTER) %{
5561 base($reg);
5562 index($ireg);
5563 scale(0x0);
5564 disp(0x0);
5565 %}
5566 %}
5567
5568 operand indIndexN(iRegN reg, iRegL lreg)
5569 %{
5570 predicate(CompressedOops::shift() == 0);
5571 constraint(ALLOC_IN_RC(ptr_reg));
5572 match(AddP (DecodeN reg) lreg);
5573 op_cost(0);
5574 format %{ "$reg, $lreg\t# narrow" %}
5575 interface(MEMORY_INTER) %{
5576 base($reg);
5577 index($lreg);
5578 scale(0x0);
5579 disp(0x0);
5580 %}
5581 %}
5582
5583 operand indOffIN(iRegN reg, immIOffset off)
5584 %{
5585 predicate(CompressedOops::shift() == 0);
5586 constraint(ALLOC_IN_RC(ptr_reg));
5587 match(AddP (DecodeN reg) off);
5588 op_cost(0);
5589 format %{ "[$reg, $off]\t# narrow" %}
5590 interface(MEMORY_INTER) %{
5591 base($reg);
5592 index(0xffffffff);
5593 scale(0x0);
5594 disp($off);
5595 %}
5596 %}
5597
5598 operand indOffLN(iRegN reg, immLOffset off)
5599 %{
5600 predicate(CompressedOops::shift() == 0);
5601 constraint(ALLOC_IN_RC(ptr_reg));
5602 match(AddP (DecodeN reg) off);
5603 op_cost(0);
5604 format %{ "[$reg, $off]\t# narrow" %}
5605 interface(MEMORY_INTER) %{
5606 base($reg);
5607 index(0xffffffff);
5608 scale(0x0);
5609 disp($off);
5610 %}
5611 %}
5612
5613
5614 //----------Special Memory Operands--------------------------------------------
5615 // Stack Slot Operand - This operand is used for loading and storing temporary
5616 // values on the stack where a match requires a value to
5617 // flow through memory.
5618 operand stackSlotP(sRegP reg)
5619 %{
5620 constraint(ALLOC_IN_RC(stack_slots));
5621 op_cost(100);
5622 // No match rule because this operand is only generated in matching
5623 // match(RegP);
5624 format %{ "[$reg]" %}
5625 interface(MEMORY_INTER) %{
5626 base(0x1e); // RSP
5627 index(0x0); // No Index
5628 scale(0x0); // No Scale
5629 disp($reg); // Stack Offset
5630 %}
5631 %}
5632
5633 operand stackSlotI(sRegI reg)
5634 %{
5635 constraint(ALLOC_IN_RC(stack_slots));
5636 // No match rule because this operand is only generated in matching
5637 // match(RegI);
5638 format %{ "[$reg]" %}
5639 interface(MEMORY_INTER) %{
5640 base(0x1e); // RSP
5641 index(0x0); // No Index
5642 scale(0x0); // No Scale
5643 disp($reg); // Stack Offset
5644 %}
5645 %}
5646
5647 operand stackSlotF(sRegF reg)
5648 %{
5649 constraint(ALLOC_IN_RC(stack_slots));
5650 // No match rule because this operand is only generated in matching
5651 // match(RegF);
5652 format %{ "[$reg]" %}
5653 interface(MEMORY_INTER) %{
5654 base(0x1e); // RSP
5655 index(0x0); // No Index
5656 scale(0x0); // No Scale
5657 disp($reg); // Stack Offset
5658 %}
5659 %}
5660
5661 operand stackSlotD(sRegD reg)
5662 %{
5663 constraint(ALLOC_IN_RC(stack_slots));
5664 // No match rule because this operand is only generated in matching
5665 // match(RegD);
5666 format %{ "[$reg]" %}
5667 interface(MEMORY_INTER) %{
5668 base(0x1e); // RSP
5669 index(0x0); // No Index
5670 scale(0x0); // No Scale
5671 disp($reg); // Stack Offset
5672 %}
5673 %}
5674
5675 operand stackSlotL(sRegL reg)
5676 %{
5677 constraint(ALLOC_IN_RC(stack_slots));
5678 // No match rule because this operand is only generated in matching
5679 // match(RegL);
5680 format %{ "[$reg]" %}
5681 interface(MEMORY_INTER) %{
5682 base(0x1e); // RSP
5683 index(0x0); // No Index
5684 scale(0x0); // No Scale
5685 disp($reg); // Stack Offset
5686 %}
5687 %}
5688
5689 // Operands for expressing Control Flow
5690 // NOTE: Label is a predefined operand which should not be redefined in
5691 // the AD file. It is generically handled within the ADLC.
5692
5693 //----------Conditional Branch Operands----------------------------------------
5694 // Comparison Op - This is the operation of the comparison, and is limited to
5695 // the following set of codes:
5696 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5697 //
5698 // Other attributes of the comparison, such as unsignedness, are specified
5699 // by the comparison instruction that sets a condition code flags register.
5700 // That result is represented by a flags operand whose subtype is appropriate
5701 // to the unsignedness (etc.) of the comparison.
5702 //
5703 // Later, the instruction which matches both the Comparison Op (a Bool) and
5704 // the flags (produced by the Cmp) specifies the coding of the comparison op
5705 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5706
5707 // used for signed integral comparisons and fp comparisons
5708
5709 operand cmpOp()
5710 %{
5711 match(Bool);
5712
5713 format %{ "" %}
5714 interface(COND_INTER) %{
5715 equal(0x0, "eq");
5716 not_equal(0x1, "ne");
5717 less(0xb, "lt");
5718 greater_equal(0xa, "ge");
5719 less_equal(0xd, "le");
5720 greater(0xc, "gt");
5721 overflow(0x6, "vs");
5722 no_overflow(0x7, "vc");
5723 %}
5724 %}
5725
5726 // used for unsigned integral comparisons
5727
5728 operand cmpOpU()
5729 %{
5730 match(Bool);
5731
5732 format %{ "" %}
5733 interface(COND_INTER) %{
5734 equal(0x0, "eq");
5735 not_equal(0x1, "ne");
5736 less(0x3, "lo");
5737 greater_equal(0x2, "hs");
5738 less_equal(0x9, "ls");
5739 greater(0x8, "hi");
5740 overflow(0x6, "vs");
5741 no_overflow(0x7, "vc");
5742 %}
5743 %}
5744
5745 // used for certain integral comparisons which can be
5746 // converted to cbxx or tbxx instructions
5747
5748 operand cmpOpEqNe()
5749 %{
5750 match(Bool);
5751 op_cost(0);
5752 predicate(n->as_Bool()->_test._test == BoolTest::ne
5753 || n->as_Bool()->_test._test == BoolTest::eq);
5754
5755 format %{ "" %}
5756 interface(COND_INTER) %{
5757 equal(0x0, "eq");
5758 not_equal(0x1, "ne");
5759 less(0xb, "lt");
5760 greater_equal(0xa, "ge");
5761 less_equal(0xd, "le");
5762 greater(0xc, "gt");
5763 overflow(0x6, "vs");
5764 no_overflow(0x7, "vc");
5765 %}
5766 %}
5767
5768 // used for certain integral comparisons which can be
5769 // converted to cbxx or tbxx instructions
5770
5771 operand cmpOpLtGe()
5772 %{
5773 match(Bool);
5774 op_cost(0);
5775
5776 predicate(n->as_Bool()->_test._test == BoolTest::lt
5777 || n->as_Bool()->_test._test == BoolTest::ge);
5778
5779 format %{ "" %}
5780 interface(COND_INTER) %{
5781 equal(0x0, "eq");
5782 not_equal(0x1, "ne");
5783 less(0xb, "lt");
5784 greater_equal(0xa, "ge");
5785 less_equal(0xd, "le");
5786 greater(0xc, "gt");
5787 overflow(0x6, "vs");
5788 no_overflow(0x7, "vc");
5789 %}
5790 %}
5791
5792 // used for certain unsigned integral comparisons which can be
5793 // converted to cbxx or tbxx instructions
5794
5795 operand cmpOpUEqNeLeGt()
5796 %{
5797 match(Bool);
5798 op_cost(0);
5799
5800 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5801 n->as_Bool()->_test._test == BoolTest::ne ||
5802 n->as_Bool()->_test._test == BoolTest::le ||
5803 n->as_Bool()->_test._test == BoolTest::gt);
5804
5805 format %{ "" %}
5806 interface(COND_INTER) %{
5807 equal(0x0, "eq");
5808 not_equal(0x1, "ne");
5809 less(0x3, "lo");
5810 greater_equal(0x2, "hs");
5811 less_equal(0x9, "ls");
5812 greater(0x8, "hi");
5813 overflow(0x6, "vs");
5814 no_overflow(0x7, "vc");
5815 %}
5816 %}
5817
5818 // Special operand allowing long args to int ops to be truncated for free
5819
5820 operand iRegL2I(iRegL reg) %{
5821
5822 op_cost(0);
5823
5824 match(ConvL2I reg);
5825
5826 format %{ "l2i($reg)" %}
5827
5828 interface(REG_INTER)
5829 %}
5830
5831 operand iRegL2P(iRegL reg) %{
5832
5833 op_cost(0);
5834
5835 match(CastX2P reg);
5836
5837 format %{ "l2p($reg)" %}
5838
5839 interface(REG_INTER)
5840 %}
5841
5842 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5843 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5844 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5845 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5846
5847 //----------OPERAND CLASSES----------------------------------------------------
5848 // Operand Classes are groups of operands that are used as to simplify
5849 // instruction definitions by not requiring the AD writer to specify
5850 // separate instructions for every form of operand when the
5851 // instruction accepts multiple operand types with the same basic
5852 // encoding and format. The classic case of this is memory operands.
5853
5854 // memory is used to define read/write location for load/store
5855 // instruction defs. we can turn a memory op into an Address
5856
5857 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5858 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5859
5860 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5861 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5862
5863 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5864 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5865
5866 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5867 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5868
5869 // All of the memory operands. For the pipeline description.
5870 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5871 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5872 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5873
5874
5875 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5876 // operations. it allows the src to be either an iRegI or a (ConvL2I
5877 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5878 // can be elided because the 32-bit instruction will just employ the
5879 // lower 32 bits anyway.
5880 //
5881 // n.b. this does not elide all L2I conversions. if the truncated
5882 // value is consumed by more than one operation then the ConvL2I
5883 // cannot be bundled into the consuming nodes so an l2i gets planted
5884 // (actually a movw $dst $src) and the downstream instructions consume
5885 // the result of the l2i as an iRegI input. That's a shame since the
5886 // movw is actually redundant but its not too costly.
5887
5888 opclass iRegIorL2I(iRegI, iRegL2I);
5889 opclass iRegPorL2P(iRegP, iRegL2P);
5890
5891 //----------PIPELINE-----------------------------------------------------------
5892 // Rules which define the behavior of the target architectures pipeline.
5893
5894 // For specific pipelines, eg A53, define the stages of that pipeline
5895 //pipe_desc(ISS, EX1, EX2, WR);
5896 #define ISS S0
5897 #define EX1 S1
5898 #define EX2 S2
5899 #define WR S3
5900
5901 // Integer ALU reg operation
5902 pipeline %{
5903
5904 attributes %{
5905 // ARM instructions are of fixed length
5906 fixed_size_instructions; // Fixed size instructions TODO does
5907 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5908 // ARM instructions come in 32-bit word units
5909 instruction_unit_size = 4; // An instruction is 4 bytes long
5910 instruction_fetch_unit_size = 64; // The processor fetches one line
5911 instruction_fetch_units = 1; // of 64 bytes
5912 %}
5913
5914 // We don't use an actual pipeline model so don't care about resources
5915 // or description. we do use pipeline classes to introduce fixed
5916 // latencies
5917
5918 //----------RESOURCES----------------------------------------------------------
5919 // Resources are the functional units available to the machine
5920
5921 resources( INS0, INS1, INS01 = INS0 | INS1,
5922 ALU0, ALU1, ALU = ALU0 | ALU1,
5923 MAC,
5924 DIV,
5925 BRANCH,
5926 LDST,
5927 NEON_FP);
5928
5929 //----------PIPELINE DESCRIPTION-----------------------------------------------
5930 // Pipeline Description specifies the stages in the machine's pipeline
5931
5932 // Define the pipeline as a generic 6 stage pipeline
5933 pipe_desc(S0, S1, S2, S3, S4, S5);
5934
5935 //----------PIPELINE CLASSES---------------------------------------------------
5936 // Pipeline Classes describe the stages in which input and output are
5937 // referenced by the hardware pipeline.
5938
5939 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5940 %{
5941 single_instruction;
5942 src1 : S1(read);
5943 src2 : S2(read);
5944 dst : S5(write);
5945 INS01 : ISS;
5946 NEON_FP : S5;
5947 %}
5948
5949 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5950 %{
5951 single_instruction;
5952 src1 : S1(read);
5953 src2 : S2(read);
5954 dst : S5(write);
5955 INS01 : ISS;
5956 NEON_FP : S5;
5957 %}
5958
5959 pipe_class fp_uop_s(vRegF dst, vRegF src)
5960 %{
5961 single_instruction;
5962 src : S1(read);
5963 dst : S5(write);
5964 INS01 : ISS;
5965 NEON_FP : S5;
5966 %}
5967
5968 pipe_class fp_uop_d(vRegD dst, vRegD src)
5969 %{
5970 single_instruction;
5971 src : S1(read);
5972 dst : S5(write);
5973 INS01 : ISS;
5974 NEON_FP : S5;
5975 %}
5976
5977 pipe_class fp_d2f(vRegF dst, vRegD src)
5978 %{
5979 single_instruction;
5980 src : S1(read);
5981 dst : S5(write);
5982 INS01 : ISS;
5983 NEON_FP : S5;
5984 %}
5985
5986 pipe_class fp_f2d(vRegD dst, vRegF src)
5987 %{
5988 single_instruction;
5989 src : S1(read);
5990 dst : S5(write);
5991 INS01 : ISS;
5992 NEON_FP : S5;
5993 %}
5994
5995 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5996 %{
5997 single_instruction;
5998 src : S1(read);
5999 dst : S5(write);
6000 INS01 : ISS;
6001 NEON_FP : S5;
6002 %}
6003
6004 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6005 %{
6006 single_instruction;
6007 src : S1(read);
6008 dst : S5(write);
6009 INS01 : ISS;
6010 NEON_FP : S5;
6011 %}
6012
6013 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6014 %{
6015 single_instruction;
6016 src : S1(read);
6017 dst : S5(write);
6018 INS01 : ISS;
6019 NEON_FP : S5;
6020 %}
6021
6022 pipe_class fp_l2f(vRegF dst, iRegL src)
6023 %{
6024 single_instruction;
6025 src : S1(read);
6026 dst : S5(write);
6027 INS01 : ISS;
6028 NEON_FP : S5;
6029 %}
6030
6031 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6032 %{
6033 single_instruction;
6034 src : S1(read);
6035 dst : S5(write);
6036 INS01 : ISS;
6037 NEON_FP : S5;
6038 %}
6039
6040 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6041 %{
6042 single_instruction;
6043 src : S1(read);
6044 dst : S5(write);
6045 INS01 : ISS;
6046 NEON_FP : S5;
6047 %}
6048
6049 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6050 %{
6051 single_instruction;
6052 src : S1(read);
6053 dst : S5(write);
6054 INS01 : ISS;
6055 NEON_FP : S5;
6056 %}
6057
6058 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6059 %{
6060 single_instruction;
6061 src : S1(read);
6062 dst : S5(write);
6063 INS01 : ISS;
6064 NEON_FP : S5;
6065 %}
6066
6067 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6068 %{
6069 single_instruction;
6070 src1 : S1(read);
6071 src2 : S2(read);
6072 dst : S5(write);
6073 INS0 : ISS;
6074 NEON_FP : S5;
6075 %}
6076
6077 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6078 %{
6079 single_instruction;
6080 src1 : S1(read);
6081 src2 : S2(read);
6082 dst : S5(write);
6083 INS0 : ISS;
6084 NEON_FP : S5;
6085 %}
6086
6087 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6088 %{
6089 single_instruction;
6090 cr : S1(read);
6091 src1 : S1(read);
6092 src2 : S1(read);
6093 dst : S3(write);
6094 INS01 : ISS;
6095 NEON_FP : S3;
6096 %}
6097
6098 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6099 %{
6100 single_instruction;
6101 cr : S1(read);
6102 src1 : S1(read);
6103 src2 : S1(read);
6104 dst : S3(write);
6105 INS01 : ISS;
6106 NEON_FP : S3;
6107 %}
6108
6109 pipe_class fp_imm_s(vRegF dst)
6110 %{
6111 single_instruction;
6112 dst : S3(write);
6113 INS01 : ISS;
6114 NEON_FP : S3;
6115 %}
6116
6117 pipe_class fp_imm_d(vRegD dst)
6118 %{
6119 single_instruction;
6120 dst : S3(write);
6121 INS01 : ISS;
6122 NEON_FP : S3;
6123 %}
6124
6125 pipe_class fp_load_constant_s(vRegF dst)
6126 %{
6127 single_instruction;
6128 dst : S4(write);
6129 INS01 : ISS;
6130 NEON_FP : S4;
6131 %}
6132
6133 pipe_class fp_load_constant_d(vRegD dst)
6134 %{
6135 single_instruction;
6136 dst : S4(write);
6137 INS01 : ISS;
6138 NEON_FP : S4;
6139 %}
6140
6141 //------- Integer ALU operations --------------------------
6142
6143 // Integer ALU reg-reg operation
6144 // Operands needed in EX1, result generated in EX2
6145 // Eg. ADD x0, x1, x2
6146 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6147 %{
6148 single_instruction;
6149 dst : EX2(write);
6150 src1 : EX1(read);
6151 src2 : EX1(read);
6152 INS01 : ISS; // Dual issue as instruction 0 or 1
6153 ALU : EX2;
6154 %}
6155
6156 // Integer ALU reg-reg operation with constant shift
6157 // Shifted register must be available in LATE_ISS instead of EX1
6158 // Eg. ADD x0, x1, x2, LSL #2
6159 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6160 %{
6161 single_instruction;
6162 dst : EX2(write);
6163 src1 : EX1(read);
6164 src2 : ISS(read);
6165 INS01 : ISS;
6166 ALU : EX2;
6167 %}
6168
6169 // Integer ALU reg operation with constant shift
6170 // Eg. LSL x0, x1, #shift
6171 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6172 %{
6173 single_instruction;
6174 dst : EX2(write);
6175 src1 : ISS(read);
6176 INS01 : ISS;
6177 ALU : EX2;
6178 %}
6179
6180 // Integer ALU reg-reg operation with variable shift
6181 // Both operands must be available in LATE_ISS instead of EX1
6182 // Result is available in EX1 instead of EX2
6183 // Eg. LSLV x0, x1, x2
6184 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6185 %{
6186 single_instruction;
6187 dst : EX1(write);
6188 src1 : ISS(read);
6189 src2 : ISS(read);
6190 INS01 : ISS;
6191 ALU : EX1;
6192 %}
6193
6194 // Integer ALU reg-reg operation with extract
6195 // As for _vshift above, but result generated in EX2
6196 // Eg. EXTR x0, x1, x2, #N
6197 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6198 %{
6199 single_instruction;
6200 dst : EX2(write);
6201 src1 : ISS(read);
6202 src2 : ISS(read);
6203 INS1 : ISS; // Can only dual issue as Instruction 1
6204 ALU : EX1;
6205 %}
6206
6207 // Integer ALU reg operation
6208 // Eg. NEG x0, x1
6209 pipe_class ialu_reg(iRegI dst, iRegI src)
6210 %{
6211 single_instruction;
6212 dst : EX2(write);
6213 src : EX1(read);
6214 INS01 : ISS;
6215 ALU : EX2;
6216 %}
6217
6218 // Integer ALU reg mmediate operation
6219 // Eg. ADD x0, x1, #N
6220 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6221 %{
6222 single_instruction;
6223 dst : EX2(write);
6224 src1 : EX1(read);
6225 INS01 : ISS;
6226 ALU : EX2;
6227 %}
6228
6229 // Integer ALU immediate operation (no source operands)
6230 // Eg. MOV x0, #N
6231 pipe_class ialu_imm(iRegI dst)
6232 %{
6233 single_instruction;
6234 dst : EX1(write);
6235 INS01 : ISS;
6236 ALU : EX1;
6237 %}
6238
6239 //------- Compare operation -------------------------------
6240
6241 // Compare reg-reg
6242 // Eg. CMP x0, x1
6243 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6244 %{
6245 single_instruction;
6246 // fixed_latency(16);
6247 cr : EX2(write);
6248 op1 : EX1(read);
6249 op2 : EX1(read);
6250 INS01 : ISS;
6251 ALU : EX2;
6252 %}
6253
6254 // Compare reg-reg
6255 // Eg. CMP x0, #N
6256 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6257 %{
6258 single_instruction;
6259 // fixed_latency(16);
6260 cr : EX2(write);
6261 op1 : EX1(read);
6262 INS01 : ISS;
6263 ALU : EX2;
6264 %}
6265
6266 //------- Conditional instructions ------------------------
6267
6268 // Conditional no operands
6269 // Eg. CSINC x0, zr, zr, <cond>
6270 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6271 %{
6272 single_instruction;
6273 cr : EX1(read);
6274 dst : EX2(write);
6275 INS01 : ISS;
6276 ALU : EX2;
6277 %}
6278
6279 // Conditional 2 operand
6280 // EG. CSEL X0, X1, X2, <cond>
6281 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6282 %{
6283 single_instruction;
6284 cr : EX1(read);
6285 src1 : EX1(read);
6286 src2 : EX1(read);
6287 dst : EX2(write);
6288 INS01 : ISS;
6289 ALU : EX2;
6290 %}
6291
6292 // Conditional 2 operand
6293 // EG. CSEL X0, X1, X2, <cond>
6294 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6295 %{
6296 single_instruction;
6297 cr : EX1(read);
6298 src : EX1(read);
6299 dst : EX2(write);
6300 INS01 : ISS;
6301 ALU : EX2;
6302 %}
6303
6304 //------- Multiply pipeline operations --------------------
6305
6306 // Multiply reg-reg
6307 // Eg. MUL w0, w1, w2
6308 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6309 %{
6310 single_instruction;
6311 dst : WR(write);
6312 src1 : ISS(read);
6313 src2 : ISS(read);
6314 INS01 : ISS;
6315 MAC : WR;
6316 %}
6317
6318 // Multiply accumulate
6319 // Eg. MADD w0, w1, w2, w3
6320 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6321 %{
6322 single_instruction;
6323 dst : WR(write);
6324 src1 : ISS(read);
6325 src2 : ISS(read);
6326 src3 : ISS(read);
6327 INS01 : ISS;
6328 MAC : WR;
6329 %}
6330
6331 // Eg. MUL w0, w1, w2
6332 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6333 %{
6334 single_instruction;
6335 fixed_latency(3); // Maximum latency for 64 bit mul
6336 dst : WR(write);
6337 src1 : ISS(read);
6338 src2 : ISS(read);
6339 INS01 : ISS;
6340 MAC : WR;
6341 %}
6342
6343 // Multiply accumulate
6344 // Eg. MADD w0, w1, w2, w3
6345 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6346 %{
6347 single_instruction;
6348 fixed_latency(3); // Maximum latency for 64 bit mul
6349 dst : WR(write);
6350 src1 : ISS(read);
6351 src2 : ISS(read);
6352 src3 : ISS(read);
6353 INS01 : ISS;
6354 MAC : WR;
6355 %}
6356
6357 //------- Divide pipeline operations --------------------
6358
6359 // Eg. SDIV w0, w1, w2
6360 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6361 %{
6362 single_instruction;
6363 fixed_latency(8); // Maximum latency for 32 bit divide
6364 dst : WR(write);
6365 src1 : ISS(read);
6366 src2 : ISS(read);
6367 INS0 : ISS; // Can only dual issue as instruction 0
6368 DIV : WR;
6369 %}
6370
6371 // Eg. SDIV x0, x1, x2
6372 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6373 %{
6374 single_instruction;
6375 fixed_latency(16); // Maximum latency for 64 bit divide
6376 dst : WR(write);
6377 src1 : ISS(read);
6378 src2 : ISS(read);
6379 INS0 : ISS; // Can only dual issue as instruction 0
6380 DIV : WR;
6381 %}
6382
6383 //------- Load pipeline operations ------------------------
6384
6385 // Load - prefetch
6386 // Eg. PFRM <mem>
6387 pipe_class iload_prefetch(memory mem)
6388 %{
6389 single_instruction;
6390 mem : ISS(read);
6391 INS01 : ISS;
6392 LDST : WR;
6393 %}
6394
6395 // Load - reg, mem
6396 // Eg. LDR x0, <mem>
6397 pipe_class iload_reg_mem(iRegI dst, memory mem)
6398 %{
6399 single_instruction;
6400 dst : WR(write);
6401 mem : ISS(read);
6402 INS01 : ISS;
6403 LDST : WR;
6404 %}
6405
6406 // Load - reg, reg
6407 // Eg. LDR x0, [sp, x1]
6408 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6409 %{
6410 single_instruction;
6411 dst : WR(write);
6412 src : ISS(read);
6413 INS01 : ISS;
6414 LDST : WR;
6415 %}
6416
6417 //------- Store pipeline operations -----------------------
6418
6419 // Store - zr, mem
6420 // Eg. STR zr, <mem>
6421 pipe_class istore_mem(memory mem)
6422 %{
6423 single_instruction;
6424 mem : ISS(read);
6425 INS01 : ISS;
6426 LDST : WR;
6427 %}
6428
6429 // Store - reg, mem
6430 // Eg. STR x0, <mem>
6431 pipe_class istore_reg_mem(iRegI src, memory mem)
6432 %{
6433 single_instruction;
6434 mem : ISS(read);
6435 src : EX2(read);
6436 INS01 : ISS;
6437 LDST : WR;
6438 %}
6439
6440 // Store - reg, reg
6441 // Eg. STR x0, [sp, x1]
6442 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6443 %{
6444 single_instruction;
6445 dst : ISS(read);
6446 src : EX2(read);
6447 INS01 : ISS;
6448 LDST : WR;
6449 %}
6450
6451 //------- Store pipeline operations -----------------------
6452
6453 // Branch
6454 pipe_class pipe_branch()
6455 %{
6456 single_instruction;
6457 INS01 : ISS;
6458 BRANCH : EX1;
6459 %}
6460
6461 // Conditional branch
6462 pipe_class pipe_branch_cond(rFlagsReg cr)
6463 %{
6464 single_instruction;
6465 cr : EX1(read);
6466 INS01 : ISS;
6467 BRANCH : EX1;
6468 %}
6469
6470 // Compare & Branch
6471 // EG. CBZ/CBNZ
6472 pipe_class pipe_cmp_branch(iRegI op1)
6473 %{
6474 single_instruction;
6475 op1 : EX1(read);
6476 INS01 : ISS;
6477 BRANCH : EX1;
6478 %}
6479
6480 //------- Synchronisation operations ----------------------
6481
6482 // Any operation requiring serialization.
6483 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6484 pipe_class pipe_serial()
6485 %{
6486 single_instruction;
6487 force_serialization;
6488 fixed_latency(16);
6489 INS01 : ISS(2); // Cannot dual issue with any other instruction
6490 LDST : WR;
6491 %}
6492
6493 // Generic big/slow expanded idiom - also serialized
6494 pipe_class pipe_slow()
6495 %{
6496 instruction_count(10);
6497 multiple_bundles;
6498 force_serialization;
6499 fixed_latency(16);
6500 INS01 : ISS(2); // Cannot dual issue with any other instruction
6501 LDST : WR;
6502 %}
6503
6504 // Empty pipeline class
6505 pipe_class pipe_class_empty()
6506 %{
6507 single_instruction;
6508 fixed_latency(0);
6509 %}
6510
6511 // Default pipeline class.
6512 pipe_class pipe_class_default()
6513 %{
6514 single_instruction;
6515 fixed_latency(2);
6516 %}
6517
6518 // Pipeline class for compares.
6519 pipe_class pipe_class_compare()
6520 %{
6521 single_instruction;
6522 fixed_latency(16);
6523 %}
6524
6525 // Pipeline class for memory operations.
6526 pipe_class pipe_class_memory()
6527 %{
6528 single_instruction;
6529 fixed_latency(16);
6530 %}
6531
6532 // Pipeline class for call.
6533 pipe_class pipe_class_call()
6534 %{
6535 single_instruction;
6536 fixed_latency(100);
6537 %}
6538
6539 // Define the class for the Nop node.
6540 define %{
6541 MachNop = pipe_class_empty;
6542 %}
6543
6544 %}
6545 //----------INSTRUCTIONS-------------------------------------------------------
6546 //
6547 // match -- States which machine-independent subtree may be replaced
6548 // by this instruction.
6549 // ins_cost -- The estimated cost of this instruction is used by instruction
6550 // selection to identify a minimum cost tree of machine
6551 // instructions that matches a tree of machine-independent
6552 // instructions.
6553 // format -- A string providing the disassembly for this instruction.
6554 // The value of an instruction's operand may be inserted
6555 // by referring to it with a '$' prefix.
6556 // opcode -- Three instruction opcodes may be provided. These are referred
6557 // to within an encode class as $primary, $secondary, and $tertiary
6558 // rrspectively. The primary opcode is commonly used to
6559 // indicate the type of machine instruction, while secondary
6560 // and tertiary are often used for prefix options or addressing
6561 // modes.
6562 // ins_encode -- A list of encode classes with parameters. The encode class
6563 // name must have been defined in an 'enc_class' specification
6564 // in the encode section of the architecture description.
6565
6566 // ============================================================================
6567 // Memory (Load/Store) Instructions
6568
6569 // Load Instructions
6570
6571 // Load Byte (8 bit signed)
6572 instruct loadB(iRegINoSp dst, memory1 mem)
6573 %{
6574 match(Set dst (LoadB mem));
6575 predicate(!needs_acquiring_load(n));
6576
6577 ins_cost(4 * INSN_COST);
6578 format %{ "ldrsbw $dst, $mem\t# byte" %}
6579
6580 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6581
6582 ins_pipe(iload_reg_mem);
6583 %}
6584
6585 // Load Byte (8 bit signed) into long
6586 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6587 %{
6588 match(Set dst (ConvI2L (LoadB mem)));
6589 predicate(!needs_acquiring_load(n->in(1)));
6590
6591 ins_cost(4 * INSN_COST);
6592 format %{ "ldrsb $dst, $mem\t# byte" %}
6593
6594 ins_encode(aarch64_enc_ldrsb(dst, mem));
6595
6596 ins_pipe(iload_reg_mem);
6597 %}
6598
6599 // Load Byte (8 bit unsigned)
6600 instruct loadUB(iRegINoSp dst, memory1 mem)
6601 %{
6602 match(Set dst (LoadUB mem));
6603 predicate(!needs_acquiring_load(n));
6604
6605 ins_cost(4 * INSN_COST);
6606 format %{ "ldrbw $dst, $mem\t# byte" %}
6607
6608 ins_encode(aarch64_enc_ldrb(dst, mem));
6609
6610 ins_pipe(iload_reg_mem);
6611 %}
6612
6613 // Load Byte (8 bit unsigned) into long
6614 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6615 %{
6616 match(Set dst (ConvI2L (LoadUB mem)));
6617 predicate(!needs_acquiring_load(n->in(1)));
6618
6619 ins_cost(4 * INSN_COST);
6620 format %{ "ldrb $dst, $mem\t# byte" %}
6621
6622 ins_encode(aarch64_enc_ldrb(dst, mem));
6623
6624 ins_pipe(iload_reg_mem);
6625 %}
6626
6627 // Load Short (16 bit signed)
6628 instruct loadS(iRegINoSp dst, memory2 mem)
6629 %{
6630 match(Set dst (LoadS mem));
6631 predicate(!needs_acquiring_load(n));
6632
6633 ins_cost(4 * INSN_COST);
6634 format %{ "ldrshw $dst, $mem\t# short" %}
6635
6636 ins_encode(aarch64_enc_ldrshw(dst, mem));
6637
6638 ins_pipe(iload_reg_mem);
6639 %}
6640
6641 // Load Short (16 bit signed) into long
6642 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6643 %{
6644 match(Set dst (ConvI2L (LoadS mem)));
6645 predicate(!needs_acquiring_load(n->in(1)));
6646
6647 ins_cost(4 * INSN_COST);
6648 format %{ "ldrsh $dst, $mem\t# short" %}
6649
6650 ins_encode(aarch64_enc_ldrsh(dst, mem));
6651
6652 ins_pipe(iload_reg_mem);
6653 %}
6654
6655 // Load Char (16 bit unsigned)
6656 instruct loadUS(iRegINoSp dst, memory2 mem)
6657 %{
6658 match(Set dst (LoadUS mem));
6659 predicate(!needs_acquiring_load(n));
6660
6661 ins_cost(4 * INSN_COST);
6662 format %{ "ldrh $dst, $mem\t# short" %}
6663
6664 ins_encode(aarch64_enc_ldrh(dst, mem));
6665
6666 ins_pipe(iload_reg_mem);
6667 %}
6668
6669 // Load Short/Char (16 bit unsigned) into long
6670 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6671 %{
6672 match(Set dst (ConvI2L (LoadUS mem)));
6673 predicate(!needs_acquiring_load(n->in(1)));
6674
6675 ins_cost(4 * INSN_COST);
6676 format %{ "ldrh $dst, $mem\t# short" %}
6677
6678 ins_encode(aarch64_enc_ldrh(dst, mem));
6679
6680 ins_pipe(iload_reg_mem);
6681 %}
6682
6683 // Load Integer (32 bit signed)
6684 instruct loadI(iRegINoSp dst, memory4 mem)
6685 %{
6686 match(Set dst (LoadI mem));
6687 predicate(!needs_acquiring_load(n));
6688
6689 ins_cost(4 * INSN_COST);
6690 format %{ "ldrw $dst, $mem\t# int" %}
6691
6692 ins_encode(aarch64_enc_ldrw(dst, mem));
6693
6694 ins_pipe(iload_reg_mem);
6695 %}
6696
6697 // Load Integer (32 bit signed) into long
6698 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6699 %{
6700 match(Set dst (ConvI2L (LoadI mem)));
6701 predicate(!needs_acquiring_load(n->in(1)));
6702
6703 ins_cost(4 * INSN_COST);
6704 format %{ "ldrsw $dst, $mem\t# int" %}
6705
6706 ins_encode(aarch64_enc_ldrsw(dst, mem));
6707
6708 ins_pipe(iload_reg_mem);
6709 %}
6710
6711 // Load Integer (32 bit unsigned) into long
6712 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6713 %{
6714 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6715 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6716
6717 ins_cost(4 * INSN_COST);
6718 format %{ "ldrw $dst, $mem\t# int" %}
6719
6720 ins_encode(aarch64_enc_ldrw(dst, mem));
6721
6722 ins_pipe(iload_reg_mem);
6723 %}
6724
6725 // Load Long (64 bit signed)
6726 instruct loadL(iRegLNoSp dst, memory8 mem)
6727 %{
6728 match(Set dst (LoadL mem));
6729 predicate(!needs_acquiring_load(n));
6730
6731 ins_cost(4 * INSN_COST);
6732 format %{ "ldr $dst, $mem\t# int" %}
6733
6734 ins_encode(aarch64_enc_ldr(dst, mem));
6735
6736 ins_pipe(iload_reg_mem);
6737 %}
6738
6739 // Load Range
6740 instruct loadRange(iRegINoSp dst, memory4 mem)
6741 %{
6742 match(Set dst (LoadRange mem));
6743
6744 ins_cost(4 * INSN_COST);
6745 format %{ "ldrw $dst, $mem\t# range" %}
6746
6747 ins_encode(aarch64_enc_ldrw(dst, mem));
6748
6749 ins_pipe(iload_reg_mem);
6750 %}
6751
6752 // Load Pointer
6753 instruct loadP(iRegPNoSp dst, memory8 mem)
6754 %{
6755 match(Set dst (LoadP mem));
6756 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6757
6758 ins_cost(4 * INSN_COST);
6759 format %{ "ldr $dst, $mem\t# ptr" %}
6760
6761 ins_encode(aarch64_enc_ldr(dst, mem));
6762
6763 ins_pipe(iload_reg_mem);
6764 %}
6765
6766 // Load Compressed Pointer
6767 instruct loadN(iRegNNoSp dst, memory4 mem)
6768 %{
6769 match(Set dst (LoadN mem));
6770 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6771
6772 ins_cost(4 * INSN_COST);
6773 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6774
6775 ins_encode(aarch64_enc_ldrw(dst, mem));
6776
6777 ins_pipe(iload_reg_mem);
6778 %}
6779
6780 // Load Klass Pointer
6781 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6782 %{
6783 match(Set dst (LoadKlass mem));
6784 predicate(!needs_acquiring_load(n));
6785
6786 ins_cost(4 * INSN_COST);
6787 format %{ "ldr $dst, $mem\t# class" %}
6788
6789 ins_encode(aarch64_enc_ldr(dst, mem));
6790
6791 ins_pipe(iload_reg_mem);
6792 %}
6793
6794 // Load Narrow Klass Pointer
6795 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6796 %{
6797 match(Set dst (LoadNKlass mem));
6798 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6799
6800 ins_cost(4 * INSN_COST);
6801 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6802
6803 ins_encode(aarch64_enc_ldrw(dst, mem));
6804
6805 ins_pipe(iload_reg_mem);
6806 %}
6807
6808 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6809 %{
6810 match(Set dst (LoadNKlass mem));
6811 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6812
6813 ins_cost(4 * INSN_COST);
6814 format %{
6815 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6816 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6817 %}
6818 ins_encode %{
6819 // inlined aarch64_enc_ldrw
6820 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6821 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6822 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6823 %}
6824 ins_pipe(iload_reg_mem);
6825 %}
6826
6827 // Load Float
6828 instruct loadF(vRegF dst, memory4 mem)
6829 %{
6830 match(Set dst (LoadF mem));
6831 predicate(!needs_acquiring_load(n));
6832
6833 ins_cost(4 * INSN_COST);
6834 format %{ "ldrs $dst, $mem\t# float" %}
6835
6836 ins_encode( aarch64_enc_ldrs(dst, mem) );
6837
6838 ins_pipe(pipe_class_memory);
6839 %}
6840
6841 // Load Double
6842 instruct loadD(vRegD dst, memory8 mem)
6843 %{
6844 match(Set dst (LoadD mem));
6845 predicate(!needs_acquiring_load(n));
6846
6847 ins_cost(4 * INSN_COST);
6848 format %{ "ldrd $dst, $mem\t# double" %}
6849
6850 ins_encode( aarch64_enc_ldrd(dst, mem) );
6851
6852 ins_pipe(pipe_class_memory);
6853 %}
6854
6855
6856 // Load Int Constant
6857 instruct loadConI(iRegINoSp dst, immI src)
6858 %{
6859 match(Set dst src);
6860
6861 ins_cost(INSN_COST);
6862 format %{ "mov $dst, $src\t# int" %}
6863
6864 ins_encode( aarch64_enc_movw_imm(dst, src) );
6865
6866 ins_pipe(ialu_imm);
6867 %}
6868
6869 // Load Long Constant
6870 instruct loadConL(iRegLNoSp dst, immL src)
6871 %{
6872 match(Set dst src);
6873
6874 ins_cost(INSN_COST);
6875 format %{ "mov $dst, $src\t# long" %}
6876
6877 ins_encode( aarch64_enc_mov_imm(dst, src) );
6878
6879 ins_pipe(ialu_imm);
6880 %}
6881
6882 // Load Pointer Constant
6883
6884 instruct loadConP(iRegPNoSp dst, immP con)
6885 %{
6886 match(Set dst con);
6887
6888 ins_cost(INSN_COST * 4);
6889 format %{
6890 "mov $dst, $con\t# ptr"
6891 %}
6892
6893 ins_encode(aarch64_enc_mov_p(dst, con));
6894
6895 ins_pipe(ialu_imm);
6896 %}
6897
6898 // Load Null Pointer Constant
6899
6900 instruct loadConP0(iRegPNoSp dst, immP0 con)
6901 %{
6902 match(Set dst con);
6903
6904 ins_cost(INSN_COST);
6905 format %{ "mov $dst, $con\t# nullptr ptr" %}
6906
6907 ins_encode(aarch64_enc_mov_p0(dst, con));
6908
6909 ins_pipe(ialu_imm);
6910 %}
6911
6912 // Load Pointer Constant One
6913
6914 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6915 %{
6916 match(Set dst con);
6917
6918 ins_cost(INSN_COST);
6919 format %{ "mov $dst, $con\t# nullptr ptr" %}
6920
6921 ins_encode(aarch64_enc_mov_p1(dst, con));
6922
6923 ins_pipe(ialu_imm);
6924 %}
6925
6926 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6927 %{
6928 match(Set dst con);
6929
6930 ins_cost(INSN_COST);
6931 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6932
6933 ins_encode %{
6934 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6935 %}
6936
6937 ins_pipe(ialu_imm);
6938 %}
6939
6940 // Load Narrow Pointer Constant
6941
6942 instruct loadConN(iRegNNoSp dst, immN con)
6943 %{
6944 match(Set dst con);
6945
6946 ins_cost(INSN_COST * 4);
6947 format %{ "mov $dst, $con\t# compressed ptr" %}
6948
6949 ins_encode(aarch64_enc_mov_n(dst, con));
6950
6951 ins_pipe(ialu_imm);
6952 %}
6953
6954 // Load Narrow Null Pointer Constant
6955
6956 instruct loadConN0(iRegNNoSp dst, immN0 con)
6957 %{
6958 match(Set dst con);
6959
6960 ins_cost(INSN_COST);
6961 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6962
6963 ins_encode(aarch64_enc_mov_n0(dst, con));
6964
6965 ins_pipe(ialu_imm);
6966 %}
6967
6968 // Load Narrow Klass Constant
6969
6970 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6971 %{
6972 match(Set dst con);
6973
6974 ins_cost(INSN_COST);
6975 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6976
6977 ins_encode(aarch64_enc_mov_nk(dst, con));
6978
6979 ins_pipe(ialu_imm);
6980 %}
6981
6982 // Load Packed Float Constant
6983
6984 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6985 match(Set dst con);
6986 ins_cost(INSN_COST * 4);
6987 format %{ "fmovs $dst, $con"%}
6988 ins_encode %{
6989 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6990 %}
6991
6992 ins_pipe(fp_imm_s);
6993 %}
6994
6995 // Load Float Constant
6996
6997 instruct loadConF(vRegF dst, immF con) %{
6998 match(Set dst con);
6999
7000 ins_cost(INSN_COST * 4);
7001
7002 format %{
7003 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7004 %}
7005
7006 ins_encode %{
7007 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7008 %}
7009
7010 ins_pipe(fp_load_constant_s);
7011 %}
7012
7013 // Load Packed Double Constant
7014
7015 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7016 match(Set dst con);
7017 ins_cost(INSN_COST);
7018 format %{ "fmovd $dst, $con"%}
7019 ins_encode %{
7020 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7021 %}
7022
7023 ins_pipe(fp_imm_d);
7024 %}
7025
7026 // Load Double Constant
7027
7028 instruct loadConD(vRegD dst, immD con) %{
7029 match(Set dst con);
7030
7031 ins_cost(INSN_COST * 5);
7032 format %{
7033 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7034 %}
7035
7036 ins_encode %{
7037 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7038 %}
7039
7040 ins_pipe(fp_load_constant_d);
7041 %}
7042
7043 // Load Half Float Constant
7044 instruct loadConH(vRegF dst, immH con) %{
7045 match(Set dst con);
7046 format %{ "mov rscratch1, $con\n\t"
7047 "fmov $dst, rscratch1"
7048 %}
7049 ins_encode %{
7050 __ movw(rscratch1, (uint32_t)$con$$constant);
7051 __ fmovs($dst$$FloatRegister, rscratch1);
7052 %}
7053 ins_pipe(pipe_class_default);
7054 %}
7055
7056 // Store Instructions
7057
7058 // Store Byte
7059 instruct storeB(iRegIorL2I src, memory1 mem)
7060 %{
7061 match(Set mem (StoreB mem src));
7062 predicate(!needs_releasing_store(n));
7063
7064 ins_cost(INSN_COST);
7065 format %{ "strb $src, $mem\t# byte" %}
7066
7067 ins_encode(aarch64_enc_strb(src, mem));
7068
7069 ins_pipe(istore_reg_mem);
7070 %}
7071
7072
7073 instruct storeimmB0(immI0 zero, memory1 mem)
7074 %{
7075 match(Set mem (StoreB mem zero));
7076 predicate(!needs_releasing_store(n));
7077
7078 ins_cost(INSN_COST);
7079 format %{ "strb rscractch2, $mem\t# byte" %}
7080
7081 ins_encode(aarch64_enc_strb0(mem));
7082
7083 ins_pipe(istore_mem);
7084 %}
7085
7086 // Store Char/Short
7087 instruct storeC(iRegIorL2I src, memory2 mem)
7088 %{
7089 match(Set mem (StoreC mem src));
7090 predicate(!needs_releasing_store(n));
7091
7092 ins_cost(INSN_COST);
7093 format %{ "strh $src, $mem\t# short" %}
7094
7095 ins_encode(aarch64_enc_strh(src, mem));
7096
7097 ins_pipe(istore_reg_mem);
7098 %}
7099
7100 instruct storeimmC0(immI0 zero, memory2 mem)
7101 %{
7102 match(Set mem (StoreC mem zero));
7103 predicate(!needs_releasing_store(n));
7104
7105 ins_cost(INSN_COST);
7106 format %{ "strh zr, $mem\t# short" %}
7107
7108 ins_encode(aarch64_enc_strh0(mem));
7109
7110 ins_pipe(istore_mem);
7111 %}
7112
7113 // Store Integer
7114
7115 instruct storeI(iRegIorL2I src, memory4 mem)
7116 %{
7117 match(Set mem(StoreI mem src));
7118 predicate(!needs_releasing_store(n));
7119
7120 ins_cost(INSN_COST);
7121 format %{ "strw $src, $mem\t# int" %}
7122
7123 ins_encode(aarch64_enc_strw(src, mem));
7124
7125 ins_pipe(istore_reg_mem);
7126 %}
7127
7128 instruct storeimmI0(immI0 zero, memory4 mem)
7129 %{
7130 match(Set mem(StoreI mem zero));
7131 predicate(!needs_releasing_store(n));
7132
7133 ins_cost(INSN_COST);
7134 format %{ "strw zr, $mem\t# int" %}
7135
7136 ins_encode(aarch64_enc_strw0(mem));
7137
7138 ins_pipe(istore_mem);
7139 %}
7140
7141 // Store Long (64 bit signed)
7142 instruct storeL(iRegL src, memory8 mem)
7143 %{
7144 match(Set mem (StoreL mem src));
7145 predicate(!needs_releasing_store(n));
7146
7147 ins_cost(INSN_COST);
7148 format %{ "str $src, $mem\t# int" %}
7149
7150 ins_encode(aarch64_enc_str(src, mem));
7151
7152 ins_pipe(istore_reg_mem);
7153 %}
7154
7155 // Store Long (64 bit signed)
7156 instruct storeimmL0(immL0 zero, memory8 mem)
7157 %{
7158 match(Set mem (StoreL mem zero));
7159 predicate(!needs_releasing_store(n));
7160
7161 ins_cost(INSN_COST);
7162 format %{ "str zr, $mem\t# int" %}
7163
7164 ins_encode(aarch64_enc_str0(mem));
7165
7166 ins_pipe(istore_mem);
7167 %}
7168
7169 // Store Pointer
7170 instruct storeP(iRegP src, memory8 mem)
7171 %{
7172 match(Set mem (StoreP mem src));
7173 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7174
7175 ins_cost(INSN_COST);
7176 format %{ "str $src, $mem\t# ptr" %}
7177
7178 ins_encode(aarch64_enc_str(src, mem));
7179
7180 ins_pipe(istore_reg_mem);
7181 %}
7182
7183 // Store Pointer
7184 instruct storeimmP0(immP0 zero, memory8 mem)
7185 %{
7186 match(Set mem (StoreP mem zero));
7187 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7188
7189 ins_cost(INSN_COST);
7190 format %{ "str zr, $mem\t# ptr" %}
7191
7192 ins_encode(aarch64_enc_str0(mem));
7193
7194 ins_pipe(istore_mem);
7195 %}
7196
7197 // Store Compressed Pointer
7198 instruct storeN(iRegN src, memory4 mem)
7199 %{
7200 match(Set mem (StoreN mem src));
7201 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7202
7203 ins_cost(INSN_COST);
7204 format %{ "strw $src, $mem\t# compressed ptr" %}
7205
7206 ins_encode(aarch64_enc_strw(src, mem));
7207
7208 ins_pipe(istore_reg_mem);
7209 %}
7210
7211 instruct storeImmN0(immN0 zero, memory4 mem)
7212 %{
7213 match(Set mem (StoreN mem zero));
7214 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7215
7216 ins_cost(INSN_COST);
7217 format %{ "strw zr, $mem\t# compressed ptr" %}
7218
7219 ins_encode(aarch64_enc_strw0(mem));
7220
7221 ins_pipe(istore_mem);
7222 %}
7223
7224 // Store Float
7225 instruct storeF(vRegF src, memory4 mem)
7226 %{
7227 match(Set mem (StoreF mem src));
7228 predicate(!needs_releasing_store(n));
7229
7230 ins_cost(INSN_COST);
7231 format %{ "strs $src, $mem\t# float" %}
7232
7233 ins_encode( aarch64_enc_strs(src, mem) );
7234
7235 ins_pipe(pipe_class_memory);
7236 %}
7237
7238 // TODO
7239 // implement storeImmF0 and storeFImmPacked
7240
7241 // Store Double
7242 instruct storeD(vRegD src, memory8 mem)
7243 %{
7244 match(Set mem (StoreD mem src));
7245 predicate(!needs_releasing_store(n));
7246
7247 ins_cost(INSN_COST);
7248 format %{ "strd $src, $mem\t# double" %}
7249
7250 ins_encode( aarch64_enc_strd(src, mem) );
7251
7252 ins_pipe(pipe_class_memory);
7253 %}
7254
7255 // Store Compressed Klass Pointer
7256 instruct storeNKlass(iRegN src, memory4 mem)
7257 %{
7258 predicate(!needs_releasing_store(n));
7259 match(Set mem (StoreNKlass mem src));
7260
7261 ins_cost(INSN_COST);
7262 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7263
7264 ins_encode(aarch64_enc_strw(src, mem));
7265
7266 ins_pipe(istore_reg_mem);
7267 %}
7268
7269 // TODO
7270 // implement storeImmD0 and storeDImmPacked
7271
7272 // prefetch instructions
7273 // Must be safe to execute with invalid address (cannot fault).
7274
7275 instruct prefetchalloc( memory8 mem ) %{
7276 match(PrefetchAllocation mem);
7277
7278 ins_cost(INSN_COST);
7279 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7280
7281 ins_encode( aarch64_enc_prefetchw(mem) );
7282
7283 ins_pipe(iload_prefetch);
7284 %}
7285
7286 // ---------------- volatile loads and stores ----------------
7287
7288 // Load Byte (8 bit signed)
7289 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7290 %{
7291 match(Set dst (LoadB mem));
7292
7293 ins_cost(VOLATILE_REF_COST);
7294 format %{ "ldarsb $dst, $mem\t# byte" %}
7295
7296 ins_encode(aarch64_enc_ldarsb(dst, mem));
7297
7298 ins_pipe(pipe_serial);
7299 %}
7300
7301 // Load Byte (8 bit signed) into long
7302 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7303 %{
7304 match(Set dst (ConvI2L (LoadB mem)));
7305
7306 ins_cost(VOLATILE_REF_COST);
7307 format %{ "ldarsb $dst, $mem\t# byte" %}
7308
7309 ins_encode(aarch64_enc_ldarsb(dst, mem));
7310
7311 ins_pipe(pipe_serial);
7312 %}
7313
7314 // Load Byte (8 bit unsigned)
7315 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7316 %{
7317 match(Set dst (LoadUB mem));
7318
7319 ins_cost(VOLATILE_REF_COST);
7320 format %{ "ldarb $dst, $mem\t# byte" %}
7321
7322 ins_encode(aarch64_enc_ldarb(dst, mem));
7323
7324 ins_pipe(pipe_serial);
7325 %}
7326
7327 // Load Byte (8 bit unsigned) into long
7328 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7329 %{
7330 match(Set dst (ConvI2L (LoadUB mem)));
7331
7332 ins_cost(VOLATILE_REF_COST);
7333 format %{ "ldarb $dst, $mem\t# byte" %}
7334
7335 ins_encode(aarch64_enc_ldarb(dst, mem));
7336
7337 ins_pipe(pipe_serial);
7338 %}
7339
7340 // Load Short (16 bit signed)
7341 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7342 %{
7343 match(Set dst (LoadS mem));
7344
7345 ins_cost(VOLATILE_REF_COST);
7346 format %{ "ldarshw $dst, $mem\t# short" %}
7347
7348 ins_encode(aarch64_enc_ldarshw(dst, mem));
7349
7350 ins_pipe(pipe_serial);
7351 %}
7352
7353 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (LoadUS mem));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldarhw $dst, $mem\t# short" %}
7359
7360 ins_encode(aarch64_enc_ldarhw(dst, mem));
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Short/Char (16 bit unsigned) into long
7366 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set dst (ConvI2L (LoadUS mem)));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldarh $dst, $mem\t# short" %}
7372
7373 ins_encode(aarch64_enc_ldarh(dst, mem));
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 // Load Short/Char (16 bit signed) into long
7379 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7380 %{
7381 match(Set dst (ConvI2L (LoadS mem)));
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "ldarh $dst, $mem\t# short" %}
7385
7386 ins_encode(aarch64_enc_ldarsh(dst, mem));
7387
7388 ins_pipe(pipe_serial);
7389 %}
7390
7391 // Load Integer (32 bit signed)
7392 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7393 %{
7394 match(Set dst (LoadI mem));
7395
7396 ins_cost(VOLATILE_REF_COST);
7397 format %{ "ldarw $dst, $mem\t# int" %}
7398
7399 ins_encode(aarch64_enc_ldarw(dst, mem));
7400
7401 ins_pipe(pipe_serial);
7402 %}
7403
7404 // Load Integer (32 bit unsigned) into long
7405 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7406 %{
7407 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7408
7409 ins_cost(VOLATILE_REF_COST);
7410 format %{ "ldarw $dst, $mem\t# int" %}
7411
7412 ins_encode(aarch64_enc_ldarw(dst, mem));
7413
7414 ins_pipe(pipe_serial);
7415 %}
7416
7417 // Load Long (64 bit signed)
7418 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7419 %{
7420 match(Set dst (LoadL mem));
7421
7422 ins_cost(VOLATILE_REF_COST);
7423 format %{ "ldar $dst, $mem\t# int" %}
7424
7425 ins_encode(aarch64_enc_ldar(dst, mem));
7426
7427 ins_pipe(pipe_serial);
7428 %}
7429
7430 // Load Pointer
7431 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7432 %{
7433 match(Set dst (LoadP mem));
7434 predicate(n->as_Load()->barrier_data() == 0);
7435
7436 ins_cost(VOLATILE_REF_COST);
7437 format %{ "ldar $dst, $mem\t# ptr" %}
7438
7439 ins_encode(aarch64_enc_ldar(dst, mem));
7440
7441 ins_pipe(pipe_serial);
7442 %}
7443
7444 // Load Compressed Pointer
7445 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7446 %{
7447 match(Set dst (LoadN mem));
7448 predicate(n->as_Load()->barrier_data() == 0);
7449
7450 ins_cost(VOLATILE_REF_COST);
7451 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7452
7453 ins_encode(aarch64_enc_ldarw(dst, mem));
7454
7455 ins_pipe(pipe_serial);
7456 %}
7457
7458 // Load Float
7459 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7460 %{
7461 match(Set dst (LoadF mem));
7462
7463 ins_cost(VOLATILE_REF_COST);
7464 format %{ "ldars $dst, $mem\t# float" %}
7465
7466 ins_encode( aarch64_enc_fldars(dst, mem) );
7467
7468 ins_pipe(pipe_serial);
7469 %}
7470
7471 // Load Double
7472 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7473 %{
7474 match(Set dst (LoadD mem));
7475
7476 ins_cost(VOLATILE_REF_COST);
7477 format %{ "ldard $dst, $mem\t# double" %}
7478
7479 ins_encode( aarch64_enc_fldard(dst, mem) );
7480
7481 ins_pipe(pipe_serial);
7482 %}
7483
7484 // Store Byte
7485 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7486 %{
7487 match(Set mem (StoreB mem src));
7488
7489 ins_cost(VOLATILE_REF_COST);
7490 format %{ "stlrb $src, $mem\t# byte" %}
7491
7492 ins_encode(aarch64_enc_stlrb(src, mem));
7493
7494 ins_pipe(pipe_class_memory);
7495 %}
7496
7497 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7498 %{
7499 match(Set mem (StoreB mem zero));
7500
7501 ins_cost(VOLATILE_REF_COST);
7502 format %{ "stlrb zr, $mem\t# byte" %}
7503
7504 ins_encode(aarch64_enc_stlrb0(mem));
7505
7506 ins_pipe(pipe_class_memory);
7507 %}
7508
7509 // Store Char/Short
7510 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7511 %{
7512 match(Set mem (StoreC mem src));
7513
7514 ins_cost(VOLATILE_REF_COST);
7515 format %{ "stlrh $src, $mem\t# short" %}
7516
7517 ins_encode(aarch64_enc_stlrh(src, mem));
7518
7519 ins_pipe(pipe_class_memory);
7520 %}
7521
7522 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7523 %{
7524 match(Set mem (StoreC mem zero));
7525
7526 ins_cost(VOLATILE_REF_COST);
7527 format %{ "stlrh zr, $mem\t# short" %}
7528
7529 ins_encode(aarch64_enc_stlrh0(mem));
7530
7531 ins_pipe(pipe_class_memory);
7532 %}
7533
7534 // Store Integer
7535
7536 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7537 %{
7538 match(Set mem(StoreI mem src));
7539
7540 ins_cost(VOLATILE_REF_COST);
7541 format %{ "stlrw $src, $mem\t# int" %}
7542
7543 ins_encode(aarch64_enc_stlrw(src, mem));
7544
7545 ins_pipe(pipe_class_memory);
7546 %}
7547
7548 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7549 %{
7550 match(Set mem(StoreI mem zero));
7551
7552 ins_cost(VOLATILE_REF_COST);
7553 format %{ "stlrw zr, $mem\t# int" %}
7554
7555 ins_encode(aarch64_enc_stlrw0(mem));
7556
7557 ins_pipe(pipe_class_memory);
7558 %}
7559
7560 // Store Long (64 bit signed)
7561 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7562 %{
7563 match(Set mem (StoreL mem src));
7564
7565 ins_cost(VOLATILE_REF_COST);
7566 format %{ "stlr $src, $mem\t# int" %}
7567
7568 ins_encode(aarch64_enc_stlr(src, mem));
7569
7570 ins_pipe(pipe_class_memory);
7571 %}
7572
7573 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7574 %{
7575 match(Set mem (StoreL mem zero));
7576
7577 ins_cost(VOLATILE_REF_COST);
7578 format %{ "stlr zr, $mem\t# int" %}
7579
7580 ins_encode(aarch64_enc_stlr0(mem));
7581
7582 ins_pipe(pipe_class_memory);
7583 %}
7584
7585 // Store Pointer
7586 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7587 %{
7588 match(Set mem (StoreP mem src));
7589 predicate(n->as_Store()->barrier_data() == 0);
7590
7591 ins_cost(VOLATILE_REF_COST);
7592 format %{ "stlr $src, $mem\t# ptr" %}
7593
7594 ins_encode(aarch64_enc_stlr(src, mem));
7595
7596 ins_pipe(pipe_class_memory);
7597 %}
7598
7599 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7600 %{
7601 match(Set mem (StoreP mem zero));
7602 predicate(n->as_Store()->barrier_data() == 0);
7603
7604 ins_cost(VOLATILE_REF_COST);
7605 format %{ "stlr zr, $mem\t# ptr" %}
7606
7607 ins_encode(aarch64_enc_stlr0(mem));
7608
7609 ins_pipe(pipe_class_memory);
7610 %}
7611
7612 // Store Compressed Pointer
7613 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7614 %{
7615 match(Set mem (StoreN mem src));
7616 predicate(n->as_Store()->barrier_data() == 0);
7617
7618 ins_cost(VOLATILE_REF_COST);
7619 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7620
7621 ins_encode(aarch64_enc_stlrw(src, mem));
7622
7623 ins_pipe(pipe_class_memory);
7624 %}
7625
7626 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7627 %{
7628 match(Set mem (StoreN mem zero));
7629 predicate(n->as_Store()->barrier_data() == 0);
7630
7631 ins_cost(VOLATILE_REF_COST);
7632 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7633
7634 ins_encode(aarch64_enc_stlrw0(mem));
7635
7636 ins_pipe(pipe_class_memory);
7637 %}
7638
7639 // Store Float
7640 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7641 %{
7642 match(Set mem (StoreF mem src));
7643
7644 ins_cost(VOLATILE_REF_COST);
7645 format %{ "stlrs $src, $mem\t# float" %}
7646
7647 ins_encode( aarch64_enc_fstlrs(src, mem) );
7648
7649 ins_pipe(pipe_class_memory);
7650 %}
7651
7652 // TODO
7653 // implement storeImmF0 and storeFImmPacked
7654
7655 // Store Double
7656 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7657 %{
7658 match(Set mem (StoreD mem src));
7659
7660 ins_cost(VOLATILE_REF_COST);
7661 format %{ "stlrd $src, $mem\t# double" %}
7662
7663 ins_encode( aarch64_enc_fstlrd(src, mem) );
7664
7665 ins_pipe(pipe_class_memory);
7666 %}
7667
7668 // ---------------- end of volatile loads and stores ----------------
7669
7670 instruct cacheWB(indirect addr)
7671 %{
7672 predicate(VM_Version::supports_data_cache_line_flush());
7673 match(CacheWB addr);
7674
7675 ins_cost(100);
7676 format %{"cache wb $addr" %}
7677 ins_encode %{
7678 assert($addr->index_position() < 0, "should be");
7679 assert($addr$$disp == 0, "should be");
7680 __ cache_wb(Address($addr$$base$$Register, 0));
7681 %}
7682 ins_pipe(pipe_slow); // XXX
7683 %}
7684
7685 instruct cacheWBPreSync()
7686 %{
7687 predicate(VM_Version::supports_data_cache_line_flush());
7688 match(CacheWBPreSync);
7689
7690 ins_cost(100);
7691 format %{"cache wb presync" %}
7692 ins_encode %{
7693 __ cache_wbsync(true);
7694 %}
7695 ins_pipe(pipe_slow); // XXX
7696 %}
7697
7698 instruct cacheWBPostSync()
7699 %{
7700 predicate(VM_Version::supports_data_cache_line_flush());
7701 match(CacheWBPostSync);
7702
7703 ins_cost(100);
7704 format %{"cache wb postsync" %}
7705 ins_encode %{
7706 __ cache_wbsync(false);
7707 %}
7708 ins_pipe(pipe_slow); // XXX
7709 %}
7710
7711 // ============================================================================
7712 // BSWAP Instructions
7713
7714 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7715 match(Set dst (ReverseBytesI src));
7716
7717 ins_cost(INSN_COST);
7718 format %{ "revw $dst, $src" %}
7719
7720 ins_encode %{
7721 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7722 %}
7723
7724 ins_pipe(ialu_reg);
7725 %}
7726
7727 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7728 match(Set dst (ReverseBytesL src));
7729
7730 ins_cost(INSN_COST);
7731 format %{ "rev $dst, $src" %}
7732
7733 ins_encode %{
7734 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7735 %}
7736
7737 ins_pipe(ialu_reg);
7738 %}
7739
7740 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7741 match(Set dst (ReverseBytesUS src));
7742
7743 ins_cost(INSN_COST);
7744 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7745
7746 ins_encode %{
7747 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7748 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7749 %}
7750
7751 ins_pipe(ialu_reg);
7752 %}
7753
7754 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7755 match(Set dst (ReverseBytesS src));
7756
7757 ins_cost(INSN_COST);
7758 format %{ "rev16w $dst, $src\n\t"
7759 "sbfmw $dst, $dst, #0, #15" %}
7760
7761 ins_encode %{
7762 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7763 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7764 %}
7765
7766 ins_pipe(ialu_reg);
7767 %}
7768
7769 // ============================================================================
7770 // Zero Count Instructions
7771
7772 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7773 match(Set dst (CountLeadingZerosI src));
7774
7775 ins_cost(INSN_COST);
7776 format %{ "clzw $dst, $src" %}
7777 ins_encode %{
7778 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7779 %}
7780
7781 ins_pipe(ialu_reg);
7782 %}
7783
7784 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7785 match(Set dst (CountLeadingZerosL src));
7786
7787 ins_cost(INSN_COST);
7788 format %{ "clz $dst, $src" %}
7789 ins_encode %{
7790 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7791 %}
7792
7793 ins_pipe(ialu_reg);
7794 %}
7795
7796 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7797 match(Set dst (CountTrailingZerosI src));
7798
7799 ins_cost(INSN_COST * 2);
7800 format %{ "rbitw $dst, $src\n\t"
7801 "clzw $dst, $dst" %}
7802 ins_encode %{
7803 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7804 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7805 %}
7806
7807 ins_pipe(ialu_reg);
7808 %}
7809
7810 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7811 match(Set dst (CountTrailingZerosL src));
7812
7813 ins_cost(INSN_COST * 2);
7814 format %{ "rbit $dst, $src\n\t"
7815 "clz $dst, $dst" %}
7816 ins_encode %{
7817 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7818 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7819 %}
7820
7821 ins_pipe(ialu_reg);
7822 %}
7823
7824 //---------- Population Count Instructions -------------------------------------
7825 //
7826
7827 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7828 match(Set dst (PopCountI src));
7829 effect(TEMP tmp);
7830 ins_cost(INSN_COST * 13);
7831
7832 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7833 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7834 "addv $tmp, $tmp\t# vector (8B)\n\t"
7835 "mov $dst, $tmp\t# vector (1D)" %}
7836 ins_encode %{
7837 __ fmovs($tmp$$FloatRegister, $src$$Register);
7838 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7839 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7840 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7841 %}
7842
7843 ins_pipe(pipe_class_default);
7844 %}
7845
7846 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7847 match(Set dst (PopCountI (LoadI mem)));
7848 effect(TEMP tmp);
7849 ins_cost(INSN_COST * 13);
7850
7851 format %{ "ldrs $tmp, $mem\n\t"
7852 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7853 "addv $tmp, $tmp\t# vector (8B)\n\t"
7854 "mov $dst, $tmp\t# vector (1D)" %}
7855 ins_encode %{
7856 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7857 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7858 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7859 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7860 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7861 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7862 %}
7863
7864 ins_pipe(pipe_class_default);
7865 %}
7866
7867 // Note: Long.bitCount(long) returns an int.
7868 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7869 match(Set dst (PopCountL src));
7870 effect(TEMP tmp);
7871 ins_cost(INSN_COST * 13);
7872
7873 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7874 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7875 "addv $tmp, $tmp\t# vector (8B)\n\t"
7876 "mov $dst, $tmp\t# vector (1D)" %}
7877 ins_encode %{
7878 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7879 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7880 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7881 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7882 %}
7883
7884 ins_pipe(pipe_class_default);
7885 %}
7886
7887 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7888 match(Set dst (PopCountL (LoadL mem)));
7889 effect(TEMP tmp);
7890 ins_cost(INSN_COST * 13);
7891
7892 format %{ "ldrd $tmp, $mem\n\t"
7893 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7894 "addv $tmp, $tmp\t# vector (8B)\n\t"
7895 "mov $dst, $tmp\t# vector (1D)" %}
7896 ins_encode %{
7897 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7898 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7899 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7900 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7901 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7902 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7903 %}
7904
7905 ins_pipe(pipe_class_default);
7906 %}
7907
7908 // ============================================================================
7909 // VerifyVectorAlignment Instruction
7910
7911 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7912 match(Set addr (VerifyVectorAlignment addr mask));
7913 effect(KILL cr);
7914 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7915 ins_encode %{
7916 Label Lskip;
7917 // check if masked bits of addr are zero
7918 __ tst($addr$$Register, $mask$$constant);
7919 __ br(Assembler::EQ, Lskip);
7920 __ stop("verify_vector_alignment found a misaligned vector memory access");
7921 __ bind(Lskip);
7922 %}
7923 ins_pipe(pipe_slow);
7924 %}
7925
7926 // ============================================================================
7927 // MemBar Instruction
7928
7929 instruct load_fence() %{
7930 match(LoadFence);
7931 ins_cost(VOLATILE_REF_COST);
7932
7933 format %{ "load_fence" %}
7934
7935 ins_encode %{
7936 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7937 %}
7938 ins_pipe(pipe_serial);
7939 %}
7940
7941 instruct unnecessary_membar_acquire() %{
7942 predicate(unnecessary_acquire(n));
7943 match(MemBarAcquire);
7944 ins_cost(0);
7945
7946 format %{ "membar_acquire (elided)" %}
7947
7948 ins_encode %{
7949 __ block_comment("membar_acquire (elided)");
7950 %}
7951
7952 ins_pipe(pipe_class_empty);
7953 %}
7954
7955 instruct membar_acquire() %{
7956 match(MemBarAcquire);
7957 ins_cost(VOLATILE_REF_COST);
7958
7959 format %{ "membar_acquire\n\t"
7960 "dmb ishld" %}
7961
7962 ins_encode %{
7963 __ block_comment("membar_acquire");
7964 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7965 %}
7966
7967 ins_pipe(pipe_serial);
7968 %}
7969
7970
7971 instruct membar_acquire_lock() %{
7972 match(MemBarAcquireLock);
7973 ins_cost(VOLATILE_REF_COST);
7974
7975 format %{ "membar_acquire_lock (elided)" %}
7976
7977 ins_encode %{
7978 __ block_comment("membar_acquire_lock (elided)");
7979 %}
7980
7981 ins_pipe(pipe_serial);
7982 %}
7983
7984 instruct store_fence() %{
7985 match(StoreFence);
7986 ins_cost(VOLATILE_REF_COST);
7987
7988 format %{ "store_fence" %}
7989
7990 ins_encode %{
7991 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7992 %}
7993 ins_pipe(pipe_serial);
7994 %}
7995
7996 instruct unnecessary_membar_release() %{
7997 predicate(unnecessary_release(n));
7998 match(MemBarRelease);
7999 ins_cost(0);
8000
8001 format %{ "membar_release (elided)" %}
8002
8003 ins_encode %{
8004 __ block_comment("membar_release (elided)");
8005 %}
8006 ins_pipe(pipe_serial);
8007 %}
8008
8009 instruct membar_release() %{
8010 match(MemBarRelease);
8011 ins_cost(VOLATILE_REF_COST);
8012
8013 format %{ "membar_release\n\t"
8014 "dmb ishst\n\tdmb ishld" %}
8015
8016 ins_encode %{
8017 __ block_comment("membar_release");
8018 // These will be merged if AlwaysMergeDMB is enabled.
8019 __ membar(Assembler::StoreStore);
8020 __ membar(Assembler::LoadStore);
8021 %}
8022 ins_pipe(pipe_serial);
8023 %}
8024
8025 instruct membar_storestore() %{
8026 match(MemBarStoreStore);
8027 match(StoreStoreFence);
8028 ins_cost(VOLATILE_REF_COST);
8029
8030 format %{ "MEMBAR-store-store" %}
8031
8032 ins_encode %{
8033 __ membar(Assembler::StoreStore);
8034 %}
8035 ins_pipe(pipe_serial);
8036 %}
8037
8038 instruct membar_release_lock() %{
8039 match(MemBarReleaseLock);
8040 ins_cost(VOLATILE_REF_COST);
8041
8042 format %{ "membar_release_lock (elided)" %}
8043
8044 ins_encode %{
8045 __ block_comment("membar_release_lock (elided)");
8046 %}
8047
8048 ins_pipe(pipe_serial);
8049 %}
8050
8051 instruct membar_storeload() %{
8052 match(MemBarStoreLoad);
8053 ins_cost(VOLATILE_REF_COST*100);
8054
8055 format %{ "MEMBAR-store-load\n\t"
8056 "dmb ish" %}
8057
8058 ins_encode %{
8059 __ block_comment("membar_storeload");
8060 __ membar(Assembler::StoreLoad);
8061 %}
8062
8063 ins_pipe(pipe_serial);
8064 %}
8065
8066 instruct unnecessary_membar_volatile() %{
8067 predicate(unnecessary_volatile(n));
8068 match(MemBarVolatile);
8069 ins_cost(0);
8070
8071 format %{ "membar_volatile (elided)" %}
8072
8073 ins_encode %{
8074 __ block_comment("membar_volatile (elided)");
8075 %}
8076
8077 ins_pipe(pipe_serial);
8078 %}
8079
8080 instruct membar_volatile() %{
8081 match(MemBarVolatile);
8082 ins_cost(VOLATILE_REF_COST*100);
8083
8084 format %{ "membar_volatile\n\t"
8085 "dmb ish"%}
8086
8087 ins_encode %{
8088 __ block_comment("membar_volatile");
8089 __ membar(Assembler::StoreLoad);
8090 %}
8091
8092 ins_pipe(pipe_serial);
8093 %}
8094
8095 instruct membar_full() %{
8096 match(MemBarFull);
8097 ins_cost(VOLATILE_REF_COST*100);
8098
8099 format %{ "membar_full\n\t"
8100 "dmb ish" %}
8101 ins_encode %{
8102 __ block_comment("membar_full");
8103 __ membar(Assembler::AnyAny);
8104 %}
8105
8106 ins_pipe(pipe_serial);
8107 %}
8108
8109 // ============================================================================
8110 // Cast/Convert Instructions
8111
8112 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8113 match(Set dst (CastX2P src));
8114
8115 ins_cost(INSN_COST);
8116 format %{ "mov $dst, $src\t# long -> ptr" %}
8117
8118 ins_encode %{
8119 if ($dst$$reg != $src$$reg) {
8120 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8121 }
8122 %}
8123
8124 ins_pipe(ialu_reg);
8125 %}
8126
8127 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8128 match(Set dst (CastI2N src));
8129
8130 ins_cost(INSN_COST);
8131 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8132
8133 ins_encode %{
8134 if ($dst$$reg != $src$$reg) {
8135 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8136 }
8137 %}
8138
8139 ins_pipe(ialu_reg);
8140 %}
8141
8142 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8143 match(Set dst (CastP2X src));
8144
8145 ins_cost(INSN_COST);
8146 format %{ "mov $dst, $src\t# ptr -> long" %}
8147
8148 ins_encode %{
8149 if ($dst$$reg != $src$$reg) {
8150 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8151 }
8152 %}
8153
8154 ins_pipe(ialu_reg);
8155 %}
8156
8157 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8158 match(Set dst (CastP2X src));
8159
8160 ins_cost(INSN_COST);
8161 format %{ "mov $dst, $src\t# ptr -> long" %}
8162
8163 ins_encode %{
8164 if ($dst$$reg != $src$$reg) {
8165 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8166 }
8167 %}
8168
8169 ins_pipe(ialu_reg);
8170 %}
8171
8172 // Convert oop into int for vectors alignment masking
8173 instruct convP2I(iRegINoSp dst, iRegP src) %{
8174 match(Set dst (ConvL2I (CastP2X src)));
8175
8176 ins_cost(INSN_COST);
8177 format %{ "movw $dst, $src\t# ptr -> int" %}
8178 ins_encode %{
8179 __ movw($dst$$Register, $src$$Register);
8180 %}
8181
8182 ins_pipe(ialu_reg);
8183 %}
8184
8185 // Convert compressed oop into int for vectors alignment masking
8186 // in case of 32bit oops (heap < 4Gb).
8187 instruct convN2I(iRegINoSp dst, iRegN src)
8188 %{
8189 predicate(CompressedOops::shift() == 0);
8190 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8191
8192 ins_cost(INSN_COST);
8193 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8194 ins_encode %{
8195 __ movw($dst$$Register, $src$$Register);
8196 %}
8197
8198 ins_pipe(ialu_reg);
8199 %}
8200
8201
8202 // Convert oop pointer into compressed form
8203 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8204 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8205 match(Set dst (EncodeP src));
8206 effect(KILL cr);
8207 ins_cost(INSN_COST * 3);
8208 format %{ "encode_heap_oop $dst, $src" %}
8209 ins_encode %{
8210 Register s = $src$$Register;
8211 Register d = $dst$$Register;
8212 __ encode_heap_oop(d, s);
8213 %}
8214 ins_pipe(ialu_reg);
8215 %}
8216
8217 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8218 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8219 match(Set dst (EncodeP src));
8220 ins_cost(INSN_COST * 3);
8221 format %{ "encode_heap_oop_not_null $dst, $src" %}
8222 ins_encode %{
8223 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8224 %}
8225 ins_pipe(ialu_reg);
8226 %}
8227
8228 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8229 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8230 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8231 match(Set dst (DecodeN src));
8232 ins_cost(INSN_COST * 3);
8233 format %{ "decode_heap_oop $dst, $src" %}
8234 ins_encode %{
8235 Register s = $src$$Register;
8236 Register d = $dst$$Register;
8237 __ decode_heap_oop(d, s);
8238 %}
8239 ins_pipe(ialu_reg);
8240 %}
8241
8242 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8243 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8244 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8245 match(Set dst (DecodeN src));
8246 ins_cost(INSN_COST * 3);
8247 format %{ "decode_heap_oop_not_null $dst, $src" %}
8248 ins_encode %{
8249 Register s = $src$$Register;
8250 Register d = $dst$$Register;
8251 __ decode_heap_oop_not_null(d, s);
8252 %}
8253 ins_pipe(ialu_reg);
8254 %}
8255
8256 // n.b. AArch64 implementations of encode_klass_not_null and
8257 // decode_klass_not_null do not modify the flags register so, unlike
8258 // Intel, we don't kill CR as a side effect here
8259
8260 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8261 match(Set dst (EncodePKlass src));
8262
8263 ins_cost(INSN_COST * 3);
8264 format %{ "encode_klass_not_null $dst,$src" %}
8265
8266 ins_encode %{
8267 Register src_reg = as_Register($src$$reg);
8268 Register dst_reg = as_Register($dst$$reg);
8269 __ encode_klass_not_null(dst_reg, src_reg);
8270 %}
8271
8272 ins_pipe(ialu_reg);
8273 %}
8274
8275 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8276 match(Set dst (DecodeNKlass src));
8277
8278 ins_cost(INSN_COST * 3);
8279 format %{ "decode_klass_not_null $dst,$src" %}
8280
8281 ins_encode %{
8282 Register src_reg = as_Register($src$$reg);
8283 Register dst_reg = as_Register($dst$$reg);
8284 if (dst_reg != src_reg) {
8285 __ decode_klass_not_null(dst_reg, src_reg);
8286 } else {
8287 __ decode_klass_not_null(dst_reg);
8288 }
8289 %}
8290
8291 ins_pipe(ialu_reg);
8292 %}
8293
8294 instruct checkCastPP(iRegPNoSp dst)
8295 %{
8296 match(Set dst (CheckCastPP dst));
8297
8298 size(0);
8299 format %{ "# checkcastPP of $dst" %}
8300 ins_encode(/* empty encoding */);
8301 ins_pipe(pipe_class_empty);
8302 %}
8303
8304 instruct castPP(iRegPNoSp dst)
8305 %{
8306 match(Set dst (CastPP dst));
8307
8308 size(0);
8309 format %{ "# castPP of $dst" %}
8310 ins_encode(/* empty encoding */);
8311 ins_pipe(pipe_class_empty);
8312 %}
8313
8314 instruct castII(iRegI dst)
8315 %{
8316 predicate(VerifyConstraintCasts == 0);
8317 match(Set dst (CastII dst));
8318
8319 size(0);
8320 format %{ "# castII of $dst" %}
8321 ins_encode(/* empty encoding */);
8322 ins_cost(0);
8323 ins_pipe(pipe_class_empty);
8324 %}
8325
8326 instruct castII_checked(iRegI dst, rFlagsReg cr)
8327 %{
8328 predicate(VerifyConstraintCasts > 0);
8329 match(Set dst (CastII dst));
8330 effect(KILL cr);
8331
8332 format %{ "# castII_checked of $dst" %}
8333 ins_encode %{
8334 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8335 %}
8336 ins_pipe(pipe_slow);
8337 %}
8338
8339 instruct castLL(iRegL dst)
8340 %{
8341 predicate(VerifyConstraintCasts == 0);
8342 match(Set dst (CastLL dst));
8343
8344 size(0);
8345 format %{ "# castLL of $dst" %}
8346 ins_encode(/* empty encoding */);
8347 ins_cost(0);
8348 ins_pipe(pipe_class_empty);
8349 %}
8350
8351 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8352 %{
8353 predicate(VerifyConstraintCasts > 0);
8354 match(Set dst (CastLL dst));
8355 effect(KILL cr);
8356
8357 format %{ "# castLL_checked of $dst" %}
8358 ins_encode %{
8359 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8360 %}
8361 ins_pipe(pipe_slow);
8362 %}
8363
8364 instruct castHH(vRegF dst)
8365 %{
8366 match(Set dst (CastHH dst));
8367 size(0);
8368 format %{ "# castHH of $dst" %}
8369 ins_encode(/* empty encoding */);
8370 ins_cost(0);
8371 ins_pipe(pipe_class_empty);
8372 %}
8373
8374 instruct castFF(vRegF dst)
8375 %{
8376 match(Set dst (CastFF dst));
8377
8378 size(0);
8379 format %{ "# castFF of $dst" %}
8380 ins_encode(/* empty encoding */);
8381 ins_cost(0);
8382 ins_pipe(pipe_class_empty);
8383 %}
8384
8385 instruct castDD(vRegD dst)
8386 %{
8387 match(Set dst (CastDD dst));
8388
8389 size(0);
8390 format %{ "# castDD of $dst" %}
8391 ins_encode(/* empty encoding */);
8392 ins_cost(0);
8393 ins_pipe(pipe_class_empty);
8394 %}
8395
8396 instruct castVV(vReg dst)
8397 %{
8398 match(Set dst (CastVV dst));
8399
8400 size(0);
8401 format %{ "# castVV of $dst" %}
8402 ins_encode(/* empty encoding */);
8403 ins_cost(0);
8404 ins_pipe(pipe_class_empty);
8405 %}
8406
8407 instruct castVVMask(pRegGov dst)
8408 %{
8409 match(Set dst (CastVV dst));
8410
8411 size(0);
8412 format %{ "# castVV of $dst" %}
8413 ins_encode(/* empty encoding */);
8414 ins_cost(0);
8415 ins_pipe(pipe_class_empty);
8416 %}
8417
8418 // Manifest a CmpU result in an integer register.
8419 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8420 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8421 %{
8422 match(Set dst (CmpU3 src1 src2));
8423 effect(KILL flags);
8424
8425 ins_cost(INSN_COST * 3);
8426 format %{
8427 "cmpw $src1, $src2\n\t"
8428 "csetw $dst, ne\n\t"
8429 "cnegw $dst, lo\t# CmpU3(reg)"
8430 %}
8431 ins_encode %{
8432 __ cmpw($src1$$Register, $src2$$Register);
8433 __ csetw($dst$$Register, Assembler::NE);
8434 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8435 %}
8436
8437 ins_pipe(pipe_class_default);
8438 %}
8439
8440 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8441 %{
8442 match(Set dst (CmpU3 src1 src2));
8443 effect(KILL flags);
8444
8445 ins_cost(INSN_COST * 3);
8446 format %{
8447 "subsw zr, $src1, $src2\n\t"
8448 "csetw $dst, ne\n\t"
8449 "cnegw $dst, lo\t# CmpU3(imm)"
8450 %}
8451 ins_encode %{
8452 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8453 __ csetw($dst$$Register, Assembler::NE);
8454 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8455 %}
8456
8457 ins_pipe(pipe_class_default);
8458 %}
8459
8460 // Manifest a CmpUL result in an integer register.
8461 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8462 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8463 %{
8464 match(Set dst (CmpUL3 src1 src2));
8465 effect(KILL flags);
8466
8467 ins_cost(INSN_COST * 3);
8468 format %{
8469 "cmp $src1, $src2\n\t"
8470 "csetw $dst, ne\n\t"
8471 "cnegw $dst, lo\t# CmpUL3(reg)"
8472 %}
8473 ins_encode %{
8474 __ cmp($src1$$Register, $src2$$Register);
8475 __ csetw($dst$$Register, Assembler::NE);
8476 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8477 %}
8478
8479 ins_pipe(pipe_class_default);
8480 %}
8481
8482 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8483 %{
8484 match(Set dst (CmpUL3 src1 src2));
8485 effect(KILL flags);
8486
8487 ins_cost(INSN_COST * 3);
8488 format %{
8489 "subs zr, $src1, $src2\n\t"
8490 "csetw $dst, ne\n\t"
8491 "cnegw $dst, lo\t# CmpUL3(imm)"
8492 %}
8493 ins_encode %{
8494 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8495 __ csetw($dst$$Register, Assembler::NE);
8496 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8497 %}
8498
8499 ins_pipe(pipe_class_default);
8500 %}
8501
8502 // Manifest a CmpL result in an integer register.
8503 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8504 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8505 %{
8506 match(Set dst (CmpL3 src1 src2));
8507 effect(KILL flags);
8508
8509 ins_cost(INSN_COST * 3);
8510 format %{
8511 "cmp $src1, $src2\n\t"
8512 "csetw $dst, ne\n\t"
8513 "cnegw $dst, lt\t# CmpL3(reg)"
8514 %}
8515 ins_encode %{
8516 __ cmp($src1$$Register, $src2$$Register);
8517 __ csetw($dst$$Register, Assembler::NE);
8518 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8519 %}
8520
8521 ins_pipe(pipe_class_default);
8522 %}
8523
8524 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8525 %{
8526 match(Set dst (CmpL3 src1 src2));
8527 effect(KILL flags);
8528
8529 ins_cost(INSN_COST * 3);
8530 format %{
8531 "subs zr, $src1, $src2\n\t"
8532 "csetw $dst, ne\n\t"
8533 "cnegw $dst, lt\t# CmpL3(imm)"
8534 %}
8535 ins_encode %{
8536 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8537 __ csetw($dst$$Register, Assembler::NE);
8538 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8539 %}
8540
8541 ins_pipe(pipe_class_default);
8542 %}
8543
8544 // ============================================================================
8545 // Conditional Move Instructions
8546
8547 // n.b. we have identical rules for both a signed compare op (cmpOp)
8548 // and an unsigned compare op (cmpOpU). it would be nice if we could
8549 // define an op class which merged both inputs and use it to type the
8550 // argument to a single rule. unfortunatelyt his fails because the
8551 // opclass does not live up to the COND_INTER interface of its
8552 // component operands. When the generic code tries to negate the
8553 // operand it ends up running the generci Machoper::negate method
8554 // which throws a ShouldNotHappen. So, we have to provide two flavours
8555 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8556
8557 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8558 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8559
8560 ins_cost(INSN_COST * 2);
8561 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8562
8563 ins_encode %{
8564 __ cselw(as_Register($dst$$reg),
8565 as_Register($src2$$reg),
8566 as_Register($src1$$reg),
8567 (Assembler::Condition)$cmp$$cmpcode);
8568 %}
8569
8570 ins_pipe(icond_reg_reg);
8571 %}
8572
8573 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8574 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8575
8576 ins_cost(INSN_COST * 2);
8577 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8578
8579 ins_encode %{
8580 __ cselw(as_Register($dst$$reg),
8581 as_Register($src2$$reg),
8582 as_Register($src1$$reg),
8583 (Assembler::Condition)$cmp$$cmpcode);
8584 %}
8585
8586 ins_pipe(icond_reg_reg);
8587 %}
8588
8589 // special cases where one arg is zero
8590
8591 // n.b. this is selected in preference to the rule above because it
8592 // avoids loading constant 0 into a source register
8593
8594 // TODO
8595 // we ought only to be able to cull one of these variants as the ideal
8596 // transforms ought always to order the zero consistently (to left/right?)
8597
8598 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8599 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8600
8601 ins_cost(INSN_COST * 2);
8602 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8603
8604 ins_encode %{
8605 __ cselw(as_Register($dst$$reg),
8606 as_Register($src$$reg),
8607 zr,
8608 (Assembler::Condition)$cmp$$cmpcode);
8609 %}
8610
8611 ins_pipe(icond_reg);
8612 %}
8613
8614 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8615 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8616
8617 ins_cost(INSN_COST * 2);
8618 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8619
8620 ins_encode %{
8621 __ cselw(as_Register($dst$$reg),
8622 as_Register($src$$reg),
8623 zr,
8624 (Assembler::Condition)$cmp$$cmpcode);
8625 %}
8626
8627 ins_pipe(icond_reg);
8628 %}
8629
8630 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8631 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8632
8633 ins_cost(INSN_COST * 2);
8634 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8635
8636 ins_encode %{
8637 __ cselw(as_Register($dst$$reg),
8638 zr,
8639 as_Register($src$$reg),
8640 (Assembler::Condition)$cmp$$cmpcode);
8641 %}
8642
8643 ins_pipe(icond_reg);
8644 %}
8645
8646 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8647 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8648
8649 ins_cost(INSN_COST * 2);
8650 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8651
8652 ins_encode %{
8653 __ cselw(as_Register($dst$$reg),
8654 zr,
8655 as_Register($src$$reg),
8656 (Assembler::Condition)$cmp$$cmpcode);
8657 %}
8658
8659 ins_pipe(icond_reg);
8660 %}
8661
8662 // special case for creating a boolean 0 or 1
8663
8664 // n.b. this is selected in preference to the rule above because it
8665 // avoids loading constants 0 and 1 into a source register
8666
8667 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8668 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8669
8670 ins_cost(INSN_COST * 2);
8671 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8672
8673 ins_encode %{
8674 // equivalently
8675 // cset(as_Register($dst$$reg),
8676 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8677 __ csincw(as_Register($dst$$reg),
8678 zr,
8679 zr,
8680 (Assembler::Condition)$cmp$$cmpcode);
8681 %}
8682
8683 ins_pipe(icond_none);
8684 %}
8685
8686 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8687 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8688
8689 ins_cost(INSN_COST * 2);
8690 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8691
8692 ins_encode %{
8693 // equivalently
8694 // cset(as_Register($dst$$reg),
8695 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8696 __ csincw(as_Register($dst$$reg),
8697 zr,
8698 zr,
8699 (Assembler::Condition)$cmp$$cmpcode);
8700 %}
8701
8702 ins_pipe(icond_none);
8703 %}
8704
8705 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8706 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8707
8708 ins_cost(INSN_COST * 2);
8709 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8710
8711 ins_encode %{
8712 __ csel(as_Register($dst$$reg),
8713 as_Register($src2$$reg),
8714 as_Register($src1$$reg),
8715 (Assembler::Condition)$cmp$$cmpcode);
8716 %}
8717
8718 ins_pipe(icond_reg_reg);
8719 %}
8720
8721 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8722 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8723
8724 ins_cost(INSN_COST * 2);
8725 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8726
8727 ins_encode %{
8728 __ csel(as_Register($dst$$reg),
8729 as_Register($src2$$reg),
8730 as_Register($src1$$reg),
8731 (Assembler::Condition)$cmp$$cmpcode);
8732 %}
8733
8734 ins_pipe(icond_reg_reg);
8735 %}
8736
8737 // special cases where one arg is zero
8738
8739 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8740 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8741
8742 ins_cost(INSN_COST * 2);
8743 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8744
8745 ins_encode %{
8746 __ csel(as_Register($dst$$reg),
8747 zr,
8748 as_Register($src$$reg),
8749 (Assembler::Condition)$cmp$$cmpcode);
8750 %}
8751
8752 ins_pipe(icond_reg);
8753 %}
8754
8755 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8756 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8757
8758 ins_cost(INSN_COST * 2);
8759 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8760
8761 ins_encode %{
8762 __ csel(as_Register($dst$$reg),
8763 zr,
8764 as_Register($src$$reg),
8765 (Assembler::Condition)$cmp$$cmpcode);
8766 %}
8767
8768 ins_pipe(icond_reg);
8769 %}
8770
8771 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8772 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8773
8774 ins_cost(INSN_COST * 2);
8775 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8776
8777 ins_encode %{
8778 __ csel(as_Register($dst$$reg),
8779 as_Register($src$$reg),
8780 zr,
8781 (Assembler::Condition)$cmp$$cmpcode);
8782 %}
8783
8784 ins_pipe(icond_reg);
8785 %}
8786
8787 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8788 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8789
8790 ins_cost(INSN_COST * 2);
8791 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8792
8793 ins_encode %{
8794 __ csel(as_Register($dst$$reg),
8795 as_Register($src$$reg),
8796 zr,
8797 (Assembler::Condition)$cmp$$cmpcode);
8798 %}
8799
8800 ins_pipe(icond_reg);
8801 %}
8802
8803 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8804 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8805
8806 ins_cost(INSN_COST * 2);
8807 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8808
8809 ins_encode %{
8810 __ csel(as_Register($dst$$reg),
8811 as_Register($src2$$reg),
8812 as_Register($src1$$reg),
8813 (Assembler::Condition)$cmp$$cmpcode);
8814 %}
8815
8816 ins_pipe(icond_reg_reg);
8817 %}
8818
8819 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8820 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8821
8822 ins_cost(INSN_COST * 2);
8823 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8824
8825 ins_encode %{
8826 __ csel(as_Register($dst$$reg),
8827 as_Register($src2$$reg),
8828 as_Register($src1$$reg),
8829 (Assembler::Condition)$cmp$$cmpcode);
8830 %}
8831
8832 ins_pipe(icond_reg_reg);
8833 %}
8834
8835 // special cases where one arg is zero
8836
8837 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8838 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8839
8840 ins_cost(INSN_COST * 2);
8841 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8842
8843 ins_encode %{
8844 __ csel(as_Register($dst$$reg),
8845 zr,
8846 as_Register($src$$reg),
8847 (Assembler::Condition)$cmp$$cmpcode);
8848 %}
8849
8850 ins_pipe(icond_reg);
8851 %}
8852
8853 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8854 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8855
8856 ins_cost(INSN_COST * 2);
8857 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8858
8859 ins_encode %{
8860 __ csel(as_Register($dst$$reg),
8861 zr,
8862 as_Register($src$$reg),
8863 (Assembler::Condition)$cmp$$cmpcode);
8864 %}
8865
8866 ins_pipe(icond_reg);
8867 %}
8868
8869 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8870 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8871
8872 ins_cost(INSN_COST * 2);
8873 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8874
8875 ins_encode %{
8876 __ csel(as_Register($dst$$reg),
8877 as_Register($src$$reg),
8878 zr,
8879 (Assembler::Condition)$cmp$$cmpcode);
8880 %}
8881
8882 ins_pipe(icond_reg);
8883 %}
8884
8885 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8886 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8887
8888 ins_cost(INSN_COST * 2);
8889 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8890
8891 ins_encode %{
8892 __ csel(as_Register($dst$$reg),
8893 as_Register($src$$reg),
8894 zr,
8895 (Assembler::Condition)$cmp$$cmpcode);
8896 %}
8897
8898 ins_pipe(icond_reg);
8899 %}
8900
8901 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8902 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8903
8904 ins_cost(INSN_COST * 2);
8905 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8906
8907 ins_encode %{
8908 __ cselw(as_Register($dst$$reg),
8909 as_Register($src2$$reg),
8910 as_Register($src1$$reg),
8911 (Assembler::Condition)$cmp$$cmpcode);
8912 %}
8913
8914 ins_pipe(icond_reg_reg);
8915 %}
8916
8917 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8918 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8919
8920 ins_cost(INSN_COST * 2);
8921 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8922
8923 ins_encode %{
8924 __ cselw(as_Register($dst$$reg),
8925 as_Register($src2$$reg),
8926 as_Register($src1$$reg),
8927 (Assembler::Condition)$cmp$$cmpcode);
8928 %}
8929
8930 ins_pipe(icond_reg_reg);
8931 %}
8932
8933 // special cases where one arg is zero
8934
8935 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8936 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8937
8938 ins_cost(INSN_COST * 2);
8939 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8940
8941 ins_encode %{
8942 __ cselw(as_Register($dst$$reg),
8943 zr,
8944 as_Register($src$$reg),
8945 (Assembler::Condition)$cmp$$cmpcode);
8946 %}
8947
8948 ins_pipe(icond_reg);
8949 %}
8950
8951 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8952 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8953
8954 ins_cost(INSN_COST * 2);
8955 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8956
8957 ins_encode %{
8958 __ cselw(as_Register($dst$$reg),
8959 zr,
8960 as_Register($src$$reg),
8961 (Assembler::Condition)$cmp$$cmpcode);
8962 %}
8963
8964 ins_pipe(icond_reg);
8965 %}
8966
8967 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8968 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8969
8970 ins_cost(INSN_COST * 2);
8971 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8972
8973 ins_encode %{
8974 __ cselw(as_Register($dst$$reg),
8975 as_Register($src$$reg),
8976 zr,
8977 (Assembler::Condition)$cmp$$cmpcode);
8978 %}
8979
8980 ins_pipe(icond_reg);
8981 %}
8982
8983 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8984 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8985
8986 ins_cost(INSN_COST * 2);
8987 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8988
8989 ins_encode %{
8990 __ cselw(as_Register($dst$$reg),
8991 as_Register($src$$reg),
8992 zr,
8993 (Assembler::Condition)$cmp$$cmpcode);
8994 %}
8995
8996 ins_pipe(icond_reg);
8997 %}
8998
8999 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9000 %{
9001 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9002
9003 ins_cost(INSN_COST * 3);
9004
9005 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9006 ins_encode %{
9007 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9008 __ fcsels(as_FloatRegister($dst$$reg),
9009 as_FloatRegister($src2$$reg),
9010 as_FloatRegister($src1$$reg),
9011 cond);
9012 %}
9013
9014 ins_pipe(fp_cond_reg_reg_s);
9015 %}
9016
9017 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9018 %{
9019 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9020
9021 ins_cost(INSN_COST * 3);
9022
9023 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9024 ins_encode %{
9025 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9026 __ fcsels(as_FloatRegister($dst$$reg),
9027 as_FloatRegister($src2$$reg),
9028 as_FloatRegister($src1$$reg),
9029 cond);
9030 %}
9031
9032 ins_pipe(fp_cond_reg_reg_s);
9033 %}
9034
9035 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9036 %{
9037 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9038
9039 ins_cost(INSN_COST * 3);
9040
9041 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9042 ins_encode %{
9043 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9044 __ fcseld(as_FloatRegister($dst$$reg),
9045 as_FloatRegister($src2$$reg),
9046 as_FloatRegister($src1$$reg),
9047 cond);
9048 %}
9049
9050 ins_pipe(fp_cond_reg_reg_d);
9051 %}
9052
9053 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9054 %{
9055 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9056
9057 ins_cost(INSN_COST * 3);
9058
9059 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9060 ins_encode %{
9061 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9062 __ fcseld(as_FloatRegister($dst$$reg),
9063 as_FloatRegister($src2$$reg),
9064 as_FloatRegister($src1$$reg),
9065 cond);
9066 %}
9067
9068 ins_pipe(fp_cond_reg_reg_d);
9069 %}
9070
9071 // ============================================================================
9072 // Arithmetic Instructions
9073 //
9074
9075 // Integer Addition
9076
9077 // TODO
9078 // these currently employ operations which do not set CR and hence are
9079 // not flagged as killing CR but we would like to isolate the cases
9080 // where we want to set flags from those where we don't. need to work
9081 // out how to do that.
9082
9083 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9084 match(Set dst (AddI src1 src2));
9085
9086 ins_cost(INSN_COST);
9087 format %{ "addw $dst, $src1, $src2" %}
9088
9089 ins_encode %{
9090 __ addw(as_Register($dst$$reg),
9091 as_Register($src1$$reg),
9092 as_Register($src2$$reg));
9093 %}
9094
9095 ins_pipe(ialu_reg_reg);
9096 %}
9097
9098 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9099 match(Set dst (AddI src1 src2));
9100
9101 ins_cost(INSN_COST);
9102 format %{ "addw $dst, $src1, $src2" %}
9103
9104 // use opcode to indicate that this is an add not a sub
9105 opcode(0x0);
9106
9107 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9108
9109 ins_pipe(ialu_reg_imm);
9110 %}
9111
9112 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9113 match(Set dst (AddI (ConvL2I src1) src2));
9114
9115 ins_cost(INSN_COST);
9116 format %{ "addw $dst, $src1, $src2" %}
9117
9118 // use opcode to indicate that this is an add not a sub
9119 opcode(0x0);
9120
9121 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9122
9123 ins_pipe(ialu_reg_imm);
9124 %}
9125
9126 // Pointer Addition
9127 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9128 match(Set dst (AddP src1 src2));
9129
9130 ins_cost(INSN_COST);
9131 format %{ "add $dst, $src1, $src2\t# ptr" %}
9132
9133 ins_encode %{
9134 __ add(as_Register($dst$$reg),
9135 as_Register($src1$$reg),
9136 as_Register($src2$$reg));
9137 %}
9138
9139 ins_pipe(ialu_reg_reg);
9140 %}
9141
9142 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9143 match(Set dst (AddP src1 (ConvI2L src2)));
9144
9145 ins_cost(1.9 * INSN_COST);
9146 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9147
9148 ins_encode %{
9149 __ add(as_Register($dst$$reg),
9150 as_Register($src1$$reg),
9151 as_Register($src2$$reg), ext::sxtw);
9152 %}
9153
9154 ins_pipe(ialu_reg_reg);
9155 %}
9156
9157 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9158 match(Set dst (AddP src1 (LShiftL src2 scale)));
9159
9160 ins_cost(1.9 * INSN_COST);
9161 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9162
9163 ins_encode %{
9164 __ lea(as_Register($dst$$reg),
9165 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9166 Address::lsl($scale$$constant)));
9167 %}
9168
9169 ins_pipe(ialu_reg_reg_shift);
9170 %}
9171
9172 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9173 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9174
9175 ins_cost(1.9 * INSN_COST);
9176 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9177
9178 ins_encode %{
9179 __ lea(as_Register($dst$$reg),
9180 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9181 Address::sxtw($scale$$constant)));
9182 %}
9183
9184 ins_pipe(ialu_reg_reg_shift);
9185 %}
9186
9187 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9188 match(Set dst (LShiftL (ConvI2L src) scale));
9189
9190 ins_cost(INSN_COST);
9191 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9192
9193 ins_encode %{
9194 __ sbfiz(as_Register($dst$$reg),
9195 as_Register($src$$reg),
9196 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9197 %}
9198
9199 ins_pipe(ialu_reg_shift);
9200 %}
9201
9202 // Pointer Immediate Addition
9203 // n.b. this needs to be more expensive than using an indirect memory
9204 // operand
9205 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9206 match(Set dst (AddP src1 src2));
9207
9208 ins_cost(INSN_COST);
9209 format %{ "add $dst, $src1, $src2\t# ptr" %}
9210
9211 // use opcode to indicate that this is an add not a sub
9212 opcode(0x0);
9213
9214 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9215
9216 ins_pipe(ialu_reg_imm);
9217 %}
9218
9219 // Long Addition
9220 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9221
9222 match(Set dst (AddL src1 src2));
9223
9224 ins_cost(INSN_COST);
9225 format %{ "add $dst, $src1, $src2" %}
9226
9227 ins_encode %{
9228 __ add(as_Register($dst$$reg),
9229 as_Register($src1$$reg),
9230 as_Register($src2$$reg));
9231 %}
9232
9233 ins_pipe(ialu_reg_reg);
9234 %}
9235
9236 // No constant pool entries requiredLong Immediate Addition.
9237 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9238 match(Set dst (AddL src1 src2));
9239
9240 ins_cost(INSN_COST);
9241 format %{ "add $dst, $src1, $src2" %}
9242
9243 // use opcode to indicate that this is an add not a sub
9244 opcode(0x0);
9245
9246 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9247
9248 ins_pipe(ialu_reg_imm);
9249 %}
9250
9251 // Integer Subtraction
9252 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9253 match(Set dst (SubI src1 src2));
9254
9255 ins_cost(INSN_COST);
9256 format %{ "subw $dst, $src1, $src2" %}
9257
9258 ins_encode %{
9259 __ subw(as_Register($dst$$reg),
9260 as_Register($src1$$reg),
9261 as_Register($src2$$reg));
9262 %}
9263
9264 ins_pipe(ialu_reg_reg);
9265 %}
9266
9267 // Immediate Subtraction
9268 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9269 match(Set dst (SubI src1 src2));
9270
9271 ins_cost(INSN_COST);
9272 format %{ "subw $dst, $src1, $src2" %}
9273
9274 // use opcode to indicate that this is a sub not an add
9275 opcode(0x1);
9276
9277 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9278
9279 ins_pipe(ialu_reg_imm);
9280 %}
9281
9282 // Long Subtraction
9283 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9284
9285 match(Set dst (SubL src1 src2));
9286
9287 ins_cost(INSN_COST);
9288 format %{ "sub $dst, $src1, $src2" %}
9289
9290 ins_encode %{
9291 __ sub(as_Register($dst$$reg),
9292 as_Register($src1$$reg),
9293 as_Register($src2$$reg));
9294 %}
9295
9296 ins_pipe(ialu_reg_reg);
9297 %}
9298
9299 // No constant pool entries requiredLong Immediate Subtraction.
9300 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9301 match(Set dst (SubL src1 src2));
9302
9303 ins_cost(INSN_COST);
9304 format %{ "sub$dst, $src1, $src2" %}
9305
9306 // use opcode to indicate that this is a sub not an add
9307 opcode(0x1);
9308
9309 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9310
9311 ins_pipe(ialu_reg_imm);
9312 %}
9313
9314 // Integer Negation (special case for sub)
9315
9316 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9317 match(Set dst (SubI zero src));
9318
9319 ins_cost(INSN_COST);
9320 format %{ "negw $dst, $src\t# int" %}
9321
9322 ins_encode %{
9323 __ negw(as_Register($dst$$reg),
9324 as_Register($src$$reg));
9325 %}
9326
9327 ins_pipe(ialu_reg);
9328 %}
9329
9330 // Long Negation
9331
9332 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9333 match(Set dst (SubL zero src));
9334
9335 ins_cost(INSN_COST);
9336 format %{ "neg $dst, $src\t# long" %}
9337
9338 ins_encode %{
9339 __ neg(as_Register($dst$$reg),
9340 as_Register($src$$reg));
9341 %}
9342
9343 ins_pipe(ialu_reg);
9344 %}
9345
9346 // Integer Multiply
9347
9348 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9349 match(Set dst (MulI src1 src2));
9350
9351 ins_cost(INSN_COST * 3);
9352 format %{ "mulw $dst, $src1, $src2" %}
9353
9354 ins_encode %{
9355 __ mulw(as_Register($dst$$reg),
9356 as_Register($src1$$reg),
9357 as_Register($src2$$reg));
9358 %}
9359
9360 ins_pipe(imul_reg_reg);
9361 %}
9362
9363 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9364 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9365
9366 ins_cost(INSN_COST * 3);
9367 format %{ "smull $dst, $src1, $src2" %}
9368
9369 ins_encode %{
9370 __ smull(as_Register($dst$$reg),
9371 as_Register($src1$$reg),
9372 as_Register($src2$$reg));
9373 %}
9374
9375 ins_pipe(imul_reg_reg);
9376 %}
9377
9378 // Long Multiply
9379
9380 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9381 match(Set dst (MulL src1 src2));
9382
9383 ins_cost(INSN_COST * 5);
9384 format %{ "mul $dst, $src1, $src2" %}
9385
9386 ins_encode %{
9387 __ mul(as_Register($dst$$reg),
9388 as_Register($src1$$reg),
9389 as_Register($src2$$reg));
9390 %}
9391
9392 ins_pipe(lmul_reg_reg);
9393 %}
9394
9395 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9396 %{
9397 match(Set dst (MulHiL src1 src2));
9398
9399 ins_cost(INSN_COST * 7);
9400 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9401
9402 ins_encode %{
9403 __ smulh(as_Register($dst$$reg),
9404 as_Register($src1$$reg),
9405 as_Register($src2$$reg));
9406 %}
9407
9408 ins_pipe(lmul_reg_reg);
9409 %}
9410
9411 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9412 %{
9413 match(Set dst (UMulHiL src1 src2));
9414
9415 ins_cost(INSN_COST * 7);
9416 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9417
9418 ins_encode %{
9419 __ umulh(as_Register($dst$$reg),
9420 as_Register($src1$$reg),
9421 as_Register($src2$$reg));
9422 %}
9423
9424 ins_pipe(lmul_reg_reg);
9425 %}
9426
9427 // Combined Integer Multiply & Add/Sub
9428
9429 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9430 match(Set dst (AddI src3 (MulI src1 src2)));
9431
9432 ins_cost(INSN_COST * 3);
9433 format %{ "madd $dst, $src1, $src2, $src3" %}
9434
9435 ins_encode %{
9436 __ maddw(as_Register($dst$$reg),
9437 as_Register($src1$$reg),
9438 as_Register($src2$$reg),
9439 as_Register($src3$$reg));
9440 %}
9441
9442 ins_pipe(imac_reg_reg);
9443 %}
9444
9445 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9446 match(Set dst (SubI src3 (MulI src1 src2)));
9447
9448 ins_cost(INSN_COST * 3);
9449 format %{ "msub $dst, $src1, $src2, $src3" %}
9450
9451 ins_encode %{
9452 __ msubw(as_Register($dst$$reg),
9453 as_Register($src1$$reg),
9454 as_Register($src2$$reg),
9455 as_Register($src3$$reg));
9456 %}
9457
9458 ins_pipe(imac_reg_reg);
9459 %}
9460
9461 // Combined Integer Multiply & Neg
9462
9463 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9464 match(Set dst (MulI (SubI zero src1) src2));
9465
9466 ins_cost(INSN_COST * 3);
9467 format %{ "mneg $dst, $src1, $src2" %}
9468
9469 ins_encode %{
9470 __ mnegw(as_Register($dst$$reg),
9471 as_Register($src1$$reg),
9472 as_Register($src2$$reg));
9473 %}
9474
9475 ins_pipe(imac_reg_reg);
9476 %}
9477
9478 // Combined Long Multiply & Add/Sub
9479
9480 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9481 match(Set dst (AddL src3 (MulL src1 src2)));
9482
9483 ins_cost(INSN_COST * 5);
9484 format %{ "madd $dst, $src1, $src2, $src3" %}
9485
9486 ins_encode %{
9487 __ madd(as_Register($dst$$reg),
9488 as_Register($src1$$reg),
9489 as_Register($src2$$reg),
9490 as_Register($src3$$reg));
9491 %}
9492
9493 ins_pipe(lmac_reg_reg);
9494 %}
9495
9496 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9497 match(Set dst (SubL src3 (MulL src1 src2)));
9498
9499 ins_cost(INSN_COST * 5);
9500 format %{ "msub $dst, $src1, $src2, $src3" %}
9501
9502 ins_encode %{
9503 __ msub(as_Register($dst$$reg),
9504 as_Register($src1$$reg),
9505 as_Register($src2$$reg),
9506 as_Register($src3$$reg));
9507 %}
9508
9509 ins_pipe(lmac_reg_reg);
9510 %}
9511
9512 // Combined Long Multiply & Neg
9513
9514 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9515 match(Set dst (MulL (SubL zero src1) src2));
9516
9517 ins_cost(INSN_COST * 5);
9518 format %{ "mneg $dst, $src1, $src2" %}
9519
9520 ins_encode %{
9521 __ mneg(as_Register($dst$$reg),
9522 as_Register($src1$$reg),
9523 as_Register($src2$$reg));
9524 %}
9525
9526 ins_pipe(lmac_reg_reg);
9527 %}
9528
9529 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9530
9531 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9532 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9533
9534 ins_cost(INSN_COST * 3);
9535 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9536
9537 ins_encode %{
9538 __ smaddl(as_Register($dst$$reg),
9539 as_Register($src1$$reg),
9540 as_Register($src2$$reg),
9541 as_Register($src3$$reg));
9542 %}
9543
9544 ins_pipe(imac_reg_reg);
9545 %}
9546
9547 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9548 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9549
9550 ins_cost(INSN_COST * 3);
9551 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9552
9553 ins_encode %{
9554 __ smsubl(as_Register($dst$$reg),
9555 as_Register($src1$$reg),
9556 as_Register($src2$$reg),
9557 as_Register($src3$$reg));
9558 %}
9559
9560 ins_pipe(imac_reg_reg);
9561 %}
9562
9563 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9564 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9565
9566 ins_cost(INSN_COST * 3);
9567 format %{ "smnegl $dst, $src1, $src2" %}
9568
9569 ins_encode %{
9570 __ smnegl(as_Register($dst$$reg),
9571 as_Register($src1$$reg),
9572 as_Register($src2$$reg));
9573 %}
9574
9575 ins_pipe(imac_reg_reg);
9576 %}
9577
9578 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9579
9580 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9581 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9582
9583 ins_cost(INSN_COST * 5);
9584 format %{ "mulw rscratch1, $src1, $src2\n\t"
9585 "maddw $dst, $src3, $src4, rscratch1" %}
9586
9587 ins_encode %{
9588 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9589 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9590
9591 ins_pipe(imac_reg_reg);
9592 %}
9593
9594 // Integer Divide
9595
9596 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9597 match(Set dst (DivI src1 src2));
9598
9599 ins_cost(INSN_COST * 19);
9600 format %{ "sdivw $dst, $src1, $src2" %}
9601
9602 ins_encode(aarch64_enc_divw(dst, src1, src2));
9603 ins_pipe(idiv_reg_reg);
9604 %}
9605
9606 // Long Divide
9607
9608 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9609 match(Set dst (DivL src1 src2));
9610
9611 ins_cost(INSN_COST * 35);
9612 format %{ "sdiv $dst, $src1, $src2" %}
9613
9614 ins_encode(aarch64_enc_div(dst, src1, src2));
9615 ins_pipe(ldiv_reg_reg);
9616 %}
9617
9618 // Integer Remainder
9619
9620 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9621 match(Set dst (ModI src1 src2));
9622
9623 ins_cost(INSN_COST * 22);
9624 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9625 "msubw $dst, rscratch1, $src2, $src1" %}
9626
9627 ins_encode(aarch64_enc_modw(dst, src1, src2));
9628 ins_pipe(idiv_reg_reg);
9629 %}
9630
9631 // Long Remainder
9632
9633 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9634 match(Set dst (ModL src1 src2));
9635
9636 ins_cost(INSN_COST * 38);
9637 format %{ "sdiv rscratch1, $src1, $src2\n"
9638 "msub $dst, rscratch1, $src2, $src1" %}
9639
9640 ins_encode(aarch64_enc_mod(dst, src1, src2));
9641 ins_pipe(ldiv_reg_reg);
9642 %}
9643
9644 // Unsigned Integer Divide
9645
9646 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9647 match(Set dst (UDivI src1 src2));
9648
9649 ins_cost(INSN_COST * 19);
9650 format %{ "udivw $dst, $src1, $src2" %}
9651
9652 ins_encode %{
9653 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9654 %}
9655
9656 ins_pipe(idiv_reg_reg);
9657 %}
9658
9659 // Unsigned Long Divide
9660
9661 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9662 match(Set dst (UDivL src1 src2));
9663
9664 ins_cost(INSN_COST * 35);
9665 format %{ "udiv $dst, $src1, $src2" %}
9666
9667 ins_encode %{
9668 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9669 %}
9670
9671 ins_pipe(ldiv_reg_reg);
9672 %}
9673
9674 // Unsigned Integer Remainder
9675
9676 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9677 match(Set dst (UModI src1 src2));
9678
9679 ins_cost(INSN_COST * 22);
9680 format %{ "udivw rscratch1, $src1, $src2\n\t"
9681 "msubw $dst, rscratch1, $src2, $src1" %}
9682
9683 ins_encode %{
9684 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9685 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9686 %}
9687
9688 ins_pipe(idiv_reg_reg);
9689 %}
9690
9691 // Unsigned Long Remainder
9692
9693 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9694 match(Set dst (UModL src1 src2));
9695
9696 ins_cost(INSN_COST * 38);
9697 format %{ "udiv rscratch1, $src1, $src2\n"
9698 "msub $dst, rscratch1, $src2, $src1" %}
9699
9700 ins_encode %{
9701 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9702 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9703 %}
9704
9705 ins_pipe(ldiv_reg_reg);
9706 %}
9707
9708 // Integer Shifts
9709
9710 // Shift Left Register
9711 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9712 match(Set dst (LShiftI src1 src2));
9713
9714 ins_cost(INSN_COST * 2);
9715 format %{ "lslvw $dst, $src1, $src2" %}
9716
9717 ins_encode %{
9718 __ lslvw(as_Register($dst$$reg),
9719 as_Register($src1$$reg),
9720 as_Register($src2$$reg));
9721 %}
9722
9723 ins_pipe(ialu_reg_reg_vshift);
9724 %}
9725
9726 // Shift Left Immediate
9727 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9728 match(Set dst (LShiftI src1 src2));
9729
9730 ins_cost(INSN_COST);
9731 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9732
9733 ins_encode %{
9734 __ lslw(as_Register($dst$$reg),
9735 as_Register($src1$$reg),
9736 $src2$$constant & 0x1f);
9737 %}
9738
9739 ins_pipe(ialu_reg_shift);
9740 %}
9741
9742 // Shift Right Logical Register
9743 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9744 match(Set dst (URShiftI src1 src2));
9745
9746 ins_cost(INSN_COST * 2);
9747 format %{ "lsrvw $dst, $src1, $src2" %}
9748
9749 ins_encode %{
9750 __ lsrvw(as_Register($dst$$reg),
9751 as_Register($src1$$reg),
9752 as_Register($src2$$reg));
9753 %}
9754
9755 ins_pipe(ialu_reg_reg_vshift);
9756 %}
9757
9758 // Shift Right Logical Immediate
9759 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9760 match(Set dst (URShiftI src1 src2));
9761
9762 ins_cost(INSN_COST);
9763 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9764
9765 ins_encode %{
9766 __ lsrw(as_Register($dst$$reg),
9767 as_Register($src1$$reg),
9768 $src2$$constant & 0x1f);
9769 %}
9770
9771 ins_pipe(ialu_reg_shift);
9772 %}
9773
9774 // Shift Right Arithmetic Register
9775 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9776 match(Set dst (RShiftI src1 src2));
9777
9778 ins_cost(INSN_COST * 2);
9779 format %{ "asrvw $dst, $src1, $src2" %}
9780
9781 ins_encode %{
9782 __ asrvw(as_Register($dst$$reg),
9783 as_Register($src1$$reg),
9784 as_Register($src2$$reg));
9785 %}
9786
9787 ins_pipe(ialu_reg_reg_vshift);
9788 %}
9789
9790 // Shift Right Arithmetic Immediate
9791 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9792 match(Set dst (RShiftI src1 src2));
9793
9794 ins_cost(INSN_COST);
9795 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9796
9797 ins_encode %{
9798 __ asrw(as_Register($dst$$reg),
9799 as_Register($src1$$reg),
9800 $src2$$constant & 0x1f);
9801 %}
9802
9803 ins_pipe(ialu_reg_shift);
9804 %}
9805
9806 // Combined Int Mask and Right Shift (using UBFM)
9807 // TODO
9808
9809 // Long Shifts
9810
9811 // Shift Left Register
9812 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9813 match(Set dst (LShiftL src1 src2));
9814
9815 ins_cost(INSN_COST * 2);
9816 format %{ "lslv $dst, $src1, $src2" %}
9817
9818 ins_encode %{
9819 __ lslv(as_Register($dst$$reg),
9820 as_Register($src1$$reg),
9821 as_Register($src2$$reg));
9822 %}
9823
9824 ins_pipe(ialu_reg_reg_vshift);
9825 %}
9826
9827 // Shift Left Immediate
9828 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9829 match(Set dst (LShiftL src1 src2));
9830
9831 ins_cost(INSN_COST);
9832 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9833
9834 ins_encode %{
9835 __ lsl(as_Register($dst$$reg),
9836 as_Register($src1$$reg),
9837 $src2$$constant & 0x3f);
9838 %}
9839
9840 ins_pipe(ialu_reg_shift);
9841 %}
9842
9843 // Shift Right Logical Register
9844 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9845 match(Set dst (URShiftL src1 src2));
9846
9847 ins_cost(INSN_COST * 2);
9848 format %{ "lsrv $dst, $src1, $src2" %}
9849
9850 ins_encode %{
9851 __ lsrv(as_Register($dst$$reg),
9852 as_Register($src1$$reg),
9853 as_Register($src2$$reg));
9854 %}
9855
9856 ins_pipe(ialu_reg_reg_vshift);
9857 %}
9858
9859 // Shift Right Logical Immediate
9860 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9861 match(Set dst (URShiftL src1 src2));
9862
9863 ins_cost(INSN_COST);
9864 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9865
9866 ins_encode %{
9867 __ lsr(as_Register($dst$$reg),
9868 as_Register($src1$$reg),
9869 $src2$$constant & 0x3f);
9870 %}
9871
9872 ins_pipe(ialu_reg_shift);
9873 %}
9874
9875 // A special-case pattern for card table stores.
9876 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9877 match(Set dst (URShiftL (CastP2X src1) src2));
9878
9879 ins_cost(INSN_COST);
9880 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9881
9882 ins_encode %{
9883 __ lsr(as_Register($dst$$reg),
9884 as_Register($src1$$reg),
9885 $src2$$constant & 0x3f);
9886 %}
9887
9888 ins_pipe(ialu_reg_shift);
9889 %}
9890
9891 // Shift Right Arithmetic Register
9892 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9893 match(Set dst (RShiftL src1 src2));
9894
9895 ins_cost(INSN_COST * 2);
9896 format %{ "asrv $dst, $src1, $src2" %}
9897
9898 ins_encode %{
9899 __ asrv(as_Register($dst$$reg),
9900 as_Register($src1$$reg),
9901 as_Register($src2$$reg));
9902 %}
9903
9904 ins_pipe(ialu_reg_reg_vshift);
9905 %}
9906
9907 // Shift Right Arithmetic Immediate
9908 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9909 match(Set dst (RShiftL src1 src2));
9910
9911 ins_cost(INSN_COST);
9912 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9913
9914 ins_encode %{
9915 __ asr(as_Register($dst$$reg),
9916 as_Register($src1$$reg),
9917 $src2$$constant & 0x3f);
9918 %}
9919
9920 ins_pipe(ialu_reg_shift);
9921 %}
9922
9923 // BEGIN This section of the file is automatically generated. Do not edit --------------
9924 // This section is generated from aarch64_ad.m4
9925
9926 // This pattern is automatically generated from aarch64_ad.m4
9927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9928 instruct regL_not_reg(iRegLNoSp dst,
9929 iRegL src1, immL_M1 m1,
9930 rFlagsReg cr) %{
9931 match(Set dst (XorL src1 m1));
9932 ins_cost(INSN_COST);
9933 format %{ "eon $dst, $src1, zr" %}
9934
9935 ins_encode %{
9936 __ eon(as_Register($dst$$reg),
9937 as_Register($src1$$reg),
9938 zr,
9939 Assembler::LSL, 0);
9940 %}
9941
9942 ins_pipe(ialu_reg);
9943 %}
9944
9945 // This pattern is automatically generated from aarch64_ad.m4
9946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9947 instruct regI_not_reg(iRegINoSp dst,
9948 iRegIorL2I src1, immI_M1 m1,
9949 rFlagsReg cr) %{
9950 match(Set dst (XorI src1 m1));
9951 ins_cost(INSN_COST);
9952 format %{ "eonw $dst, $src1, zr" %}
9953
9954 ins_encode %{
9955 __ eonw(as_Register($dst$$reg),
9956 as_Register($src1$$reg),
9957 zr,
9958 Assembler::LSL, 0);
9959 %}
9960
9961 ins_pipe(ialu_reg);
9962 %}
9963
9964 // This pattern is automatically generated from aarch64_ad.m4
9965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9966 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9967 immI0 zero, iRegIorL2I src1, immI src2) %{
9968 match(Set dst (SubI zero (URShiftI src1 src2)));
9969
9970 ins_cost(1.9 * INSN_COST);
9971 format %{ "negw $dst, $src1, LSR $src2" %}
9972
9973 ins_encode %{
9974 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9975 Assembler::LSR, $src2$$constant & 0x1f);
9976 %}
9977
9978 ins_pipe(ialu_reg_shift);
9979 %}
9980
9981 // This pattern is automatically generated from aarch64_ad.m4
9982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9983 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9984 immI0 zero, iRegIorL2I src1, immI src2) %{
9985 match(Set dst (SubI zero (RShiftI src1 src2)));
9986
9987 ins_cost(1.9 * INSN_COST);
9988 format %{ "negw $dst, $src1, ASR $src2" %}
9989
9990 ins_encode %{
9991 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9992 Assembler::ASR, $src2$$constant & 0x1f);
9993 %}
9994
9995 ins_pipe(ialu_reg_shift);
9996 %}
9997
9998 // This pattern is automatically generated from aarch64_ad.m4
9999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10000 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10001 immI0 zero, iRegIorL2I src1, immI src2) %{
10002 match(Set dst (SubI zero (LShiftI src1 src2)));
10003
10004 ins_cost(1.9 * INSN_COST);
10005 format %{ "negw $dst, $src1, LSL $src2" %}
10006
10007 ins_encode %{
10008 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10009 Assembler::LSL, $src2$$constant & 0x1f);
10010 %}
10011
10012 ins_pipe(ialu_reg_shift);
10013 %}
10014
10015 // This pattern is automatically generated from aarch64_ad.m4
10016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10017 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10018 immL0 zero, iRegL src1, immI src2) %{
10019 match(Set dst (SubL zero (URShiftL src1 src2)));
10020
10021 ins_cost(1.9 * INSN_COST);
10022 format %{ "neg $dst, $src1, LSR $src2" %}
10023
10024 ins_encode %{
10025 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10026 Assembler::LSR, $src2$$constant & 0x3f);
10027 %}
10028
10029 ins_pipe(ialu_reg_shift);
10030 %}
10031
10032 // This pattern is automatically generated from aarch64_ad.m4
10033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10034 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10035 immL0 zero, iRegL src1, immI src2) %{
10036 match(Set dst (SubL zero (RShiftL src1 src2)));
10037
10038 ins_cost(1.9 * INSN_COST);
10039 format %{ "neg $dst, $src1, ASR $src2" %}
10040
10041 ins_encode %{
10042 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10043 Assembler::ASR, $src2$$constant & 0x3f);
10044 %}
10045
10046 ins_pipe(ialu_reg_shift);
10047 %}
10048
10049 // This pattern is automatically generated from aarch64_ad.m4
10050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10051 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10052 immL0 zero, iRegL src1, immI src2) %{
10053 match(Set dst (SubL zero (LShiftL src1 src2)));
10054
10055 ins_cost(1.9 * INSN_COST);
10056 format %{ "neg $dst, $src1, LSL $src2" %}
10057
10058 ins_encode %{
10059 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10060 Assembler::LSL, $src2$$constant & 0x3f);
10061 %}
10062
10063 ins_pipe(ialu_reg_shift);
10064 %}
10065
10066 // This pattern is automatically generated from aarch64_ad.m4
10067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10068 instruct AndI_reg_not_reg(iRegINoSp dst,
10069 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10070 match(Set dst (AndI src1 (XorI src2 m1)));
10071 ins_cost(INSN_COST);
10072 format %{ "bicw $dst, $src1, $src2" %}
10073
10074 ins_encode %{
10075 __ bicw(as_Register($dst$$reg),
10076 as_Register($src1$$reg),
10077 as_Register($src2$$reg),
10078 Assembler::LSL, 0);
10079 %}
10080
10081 ins_pipe(ialu_reg_reg);
10082 %}
10083
10084 // This pattern is automatically generated from aarch64_ad.m4
10085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10086 instruct AndL_reg_not_reg(iRegLNoSp dst,
10087 iRegL src1, iRegL src2, immL_M1 m1) %{
10088 match(Set dst (AndL src1 (XorL src2 m1)));
10089 ins_cost(INSN_COST);
10090 format %{ "bic $dst, $src1, $src2" %}
10091
10092 ins_encode %{
10093 __ bic(as_Register($dst$$reg),
10094 as_Register($src1$$reg),
10095 as_Register($src2$$reg),
10096 Assembler::LSL, 0);
10097 %}
10098
10099 ins_pipe(ialu_reg_reg);
10100 %}
10101
10102 // This pattern is automatically generated from aarch64_ad.m4
10103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10104 instruct OrI_reg_not_reg(iRegINoSp dst,
10105 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10106 match(Set dst (OrI src1 (XorI src2 m1)));
10107 ins_cost(INSN_COST);
10108 format %{ "ornw $dst, $src1, $src2" %}
10109
10110 ins_encode %{
10111 __ ornw(as_Register($dst$$reg),
10112 as_Register($src1$$reg),
10113 as_Register($src2$$reg),
10114 Assembler::LSL, 0);
10115 %}
10116
10117 ins_pipe(ialu_reg_reg);
10118 %}
10119
10120 // This pattern is automatically generated from aarch64_ad.m4
10121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10122 instruct OrL_reg_not_reg(iRegLNoSp dst,
10123 iRegL src1, iRegL src2, immL_M1 m1) %{
10124 match(Set dst (OrL src1 (XorL src2 m1)));
10125 ins_cost(INSN_COST);
10126 format %{ "orn $dst, $src1, $src2" %}
10127
10128 ins_encode %{
10129 __ orn(as_Register($dst$$reg),
10130 as_Register($src1$$reg),
10131 as_Register($src2$$reg),
10132 Assembler::LSL, 0);
10133 %}
10134
10135 ins_pipe(ialu_reg_reg);
10136 %}
10137
10138 // This pattern is automatically generated from aarch64_ad.m4
10139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10140 instruct XorI_reg_not_reg(iRegINoSp dst,
10141 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10142 match(Set dst (XorI m1 (XorI src2 src1)));
10143 ins_cost(INSN_COST);
10144 format %{ "eonw $dst, $src1, $src2" %}
10145
10146 ins_encode %{
10147 __ eonw(as_Register($dst$$reg),
10148 as_Register($src1$$reg),
10149 as_Register($src2$$reg),
10150 Assembler::LSL, 0);
10151 %}
10152
10153 ins_pipe(ialu_reg_reg);
10154 %}
10155
10156 // This pattern is automatically generated from aarch64_ad.m4
10157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10158 instruct XorL_reg_not_reg(iRegLNoSp dst,
10159 iRegL src1, iRegL src2, immL_M1 m1) %{
10160 match(Set dst (XorL m1 (XorL src2 src1)));
10161 ins_cost(INSN_COST);
10162 format %{ "eon $dst, $src1, $src2" %}
10163
10164 ins_encode %{
10165 __ eon(as_Register($dst$$reg),
10166 as_Register($src1$$reg),
10167 as_Register($src2$$reg),
10168 Assembler::LSL, 0);
10169 %}
10170
10171 ins_pipe(ialu_reg_reg);
10172 %}
10173
10174 // This pattern is automatically generated from aarch64_ad.m4
10175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10176 // val & (-1 ^ (val >>> shift)) ==> bicw
10177 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10178 iRegIorL2I src1, iRegIorL2I src2,
10179 immI src3, immI_M1 src4) %{
10180 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10181 ins_cost(1.9 * INSN_COST);
10182 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10183
10184 ins_encode %{
10185 __ bicw(as_Register($dst$$reg),
10186 as_Register($src1$$reg),
10187 as_Register($src2$$reg),
10188 Assembler::LSR,
10189 $src3$$constant & 0x1f);
10190 %}
10191
10192 ins_pipe(ialu_reg_reg_shift);
10193 %}
10194
10195 // This pattern is automatically generated from aarch64_ad.m4
10196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10197 // val & (-1 ^ (val >>> shift)) ==> bic
10198 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10199 iRegL src1, iRegL src2,
10200 immI src3, immL_M1 src4) %{
10201 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10202 ins_cost(1.9 * INSN_COST);
10203 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10204
10205 ins_encode %{
10206 __ bic(as_Register($dst$$reg),
10207 as_Register($src1$$reg),
10208 as_Register($src2$$reg),
10209 Assembler::LSR,
10210 $src3$$constant & 0x3f);
10211 %}
10212
10213 ins_pipe(ialu_reg_reg_shift);
10214 %}
10215
10216 // This pattern is automatically generated from aarch64_ad.m4
10217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10218 // val & (-1 ^ (val >> shift)) ==> bicw
10219 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10220 iRegIorL2I src1, iRegIorL2I src2,
10221 immI src3, immI_M1 src4) %{
10222 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10223 ins_cost(1.9 * INSN_COST);
10224 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10225
10226 ins_encode %{
10227 __ bicw(as_Register($dst$$reg),
10228 as_Register($src1$$reg),
10229 as_Register($src2$$reg),
10230 Assembler::ASR,
10231 $src3$$constant & 0x1f);
10232 %}
10233
10234 ins_pipe(ialu_reg_reg_shift);
10235 %}
10236
10237 // This pattern is automatically generated from aarch64_ad.m4
10238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10239 // val & (-1 ^ (val >> shift)) ==> bic
10240 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10241 iRegL src1, iRegL src2,
10242 immI src3, immL_M1 src4) %{
10243 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10244 ins_cost(1.9 * INSN_COST);
10245 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10246
10247 ins_encode %{
10248 __ bic(as_Register($dst$$reg),
10249 as_Register($src1$$reg),
10250 as_Register($src2$$reg),
10251 Assembler::ASR,
10252 $src3$$constant & 0x3f);
10253 %}
10254
10255 ins_pipe(ialu_reg_reg_shift);
10256 %}
10257
10258 // This pattern is automatically generated from aarch64_ad.m4
10259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10260 // val & (-1 ^ (val ror shift)) ==> bicw
10261 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10262 iRegIorL2I src1, iRegIorL2I src2,
10263 immI src3, immI_M1 src4) %{
10264 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10265 ins_cost(1.9 * INSN_COST);
10266 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10267
10268 ins_encode %{
10269 __ bicw(as_Register($dst$$reg),
10270 as_Register($src1$$reg),
10271 as_Register($src2$$reg),
10272 Assembler::ROR,
10273 $src3$$constant & 0x1f);
10274 %}
10275
10276 ins_pipe(ialu_reg_reg_shift);
10277 %}
10278
10279 // This pattern is automatically generated from aarch64_ad.m4
10280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10281 // val & (-1 ^ (val ror shift)) ==> bic
10282 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10283 iRegL src1, iRegL src2,
10284 immI src3, immL_M1 src4) %{
10285 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10286 ins_cost(1.9 * INSN_COST);
10287 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10288
10289 ins_encode %{
10290 __ bic(as_Register($dst$$reg),
10291 as_Register($src1$$reg),
10292 as_Register($src2$$reg),
10293 Assembler::ROR,
10294 $src3$$constant & 0x3f);
10295 %}
10296
10297 ins_pipe(ialu_reg_reg_shift);
10298 %}
10299
10300 // This pattern is automatically generated from aarch64_ad.m4
10301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10302 // val & (-1 ^ (val << shift)) ==> bicw
10303 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10304 iRegIorL2I src1, iRegIorL2I src2,
10305 immI src3, immI_M1 src4) %{
10306 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10307 ins_cost(1.9 * INSN_COST);
10308 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10309
10310 ins_encode %{
10311 __ bicw(as_Register($dst$$reg),
10312 as_Register($src1$$reg),
10313 as_Register($src2$$reg),
10314 Assembler::LSL,
10315 $src3$$constant & 0x1f);
10316 %}
10317
10318 ins_pipe(ialu_reg_reg_shift);
10319 %}
10320
10321 // This pattern is automatically generated from aarch64_ad.m4
10322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10323 // val & (-1 ^ (val << shift)) ==> bic
10324 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10325 iRegL src1, iRegL src2,
10326 immI src3, immL_M1 src4) %{
10327 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10328 ins_cost(1.9 * INSN_COST);
10329 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10330
10331 ins_encode %{
10332 __ bic(as_Register($dst$$reg),
10333 as_Register($src1$$reg),
10334 as_Register($src2$$reg),
10335 Assembler::LSL,
10336 $src3$$constant & 0x3f);
10337 %}
10338
10339 ins_pipe(ialu_reg_reg_shift);
10340 %}
10341
10342 // This pattern is automatically generated from aarch64_ad.m4
10343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10344 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10345 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10346 iRegIorL2I src1, iRegIorL2I src2,
10347 immI src3, immI_M1 src4) %{
10348 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10349 ins_cost(1.9 * INSN_COST);
10350 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10351
10352 ins_encode %{
10353 __ eonw(as_Register($dst$$reg),
10354 as_Register($src1$$reg),
10355 as_Register($src2$$reg),
10356 Assembler::LSR,
10357 $src3$$constant & 0x1f);
10358 %}
10359
10360 ins_pipe(ialu_reg_reg_shift);
10361 %}
10362
10363 // This pattern is automatically generated from aarch64_ad.m4
10364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10365 // val ^ (-1 ^ (val >>> shift)) ==> eon
10366 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10367 iRegL src1, iRegL src2,
10368 immI src3, immL_M1 src4) %{
10369 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10370 ins_cost(1.9 * INSN_COST);
10371 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10372
10373 ins_encode %{
10374 __ eon(as_Register($dst$$reg),
10375 as_Register($src1$$reg),
10376 as_Register($src2$$reg),
10377 Assembler::LSR,
10378 $src3$$constant & 0x3f);
10379 %}
10380
10381 ins_pipe(ialu_reg_reg_shift);
10382 %}
10383
10384 // This pattern is automatically generated from aarch64_ad.m4
10385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10386 // val ^ (-1 ^ (val >> shift)) ==> eonw
10387 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10388 iRegIorL2I src1, iRegIorL2I src2,
10389 immI src3, immI_M1 src4) %{
10390 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10391 ins_cost(1.9 * INSN_COST);
10392 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10393
10394 ins_encode %{
10395 __ eonw(as_Register($dst$$reg),
10396 as_Register($src1$$reg),
10397 as_Register($src2$$reg),
10398 Assembler::ASR,
10399 $src3$$constant & 0x1f);
10400 %}
10401
10402 ins_pipe(ialu_reg_reg_shift);
10403 %}
10404
10405 // This pattern is automatically generated from aarch64_ad.m4
10406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10407 // val ^ (-1 ^ (val >> shift)) ==> eon
10408 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10409 iRegL src1, iRegL src2,
10410 immI src3, immL_M1 src4) %{
10411 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10412 ins_cost(1.9 * INSN_COST);
10413 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10414
10415 ins_encode %{
10416 __ eon(as_Register($dst$$reg),
10417 as_Register($src1$$reg),
10418 as_Register($src2$$reg),
10419 Assembler::ASR,
10420 $src3$$constant & 0x3f);
10421 %}
10422
10423 ins_pipe(ialu_reg_reg_shift);
10424 %}
10425
10426 // This pattern is automatically generated from aarch64_ad.m4
10427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10428 // val ^ (-1 ^ (val ror shift)) ==> eonw
10429 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10430 iRegIorL2I src1, iRegIorL2I src2,
10431 immI src3, immI_M1 src4) %{
10432 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10433 ins_cost(1.9 * INSN_COST);
10434 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10435
10436 ins_encode %{
10437 __ eonw(as_Register($dst$$reg),
10438 as_Register($src1$$reg),
10439 as_Register($src2$$reg),
10440 Assembler::ROR,
10441 $src3$$constant & 0x1f);
10442 %}
10443
10444 ins_pipe(ialu_reg_reg_shift);
10445 %}
10446
10447 // This pattern is automatically generated from aarch64_ad.m4
10448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10449 // val ^ (-1 ^ (val ror shift)) ==> eon
10450 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10451 iRegL src1, iRegL src2,
10452 immI src3, immL_M1 src4) %{
10453 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10454 ins_cost(1.9 * INSN_COST);
10455 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10456
10457 ins_encode %{
10458 __ eon(as_Register($dst$$reg),
10459 as_Register($src1$$reg),
10460 as_Register($src2$$reg),
10461 Assembler::ROR,
10462 $src3$$constant & 0x3f);
10463 %}
10464
10465 ins_pipe(ialu_reg_reg_shift);
10466 %}
10467
10468 // This pattern is automatically generated from aarch64_ad.m4
10469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10470 // val ^ (-1 ^ (val << shift)) ==> eonw
10471 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10472 iRegIorL2I src1, iRegIorL2I src2,
10473 immI src3, immI_M1 src4) %{
10474 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10475 ins_cost(1.9 * INSN_COST);
10476 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10477
10478 ins_encode %{
10479 __ eonw(as_Register($dst$$reg),
10480 as_Register($src1$$reg),
10481 as_Register($src2$$reg),
10482 Assembler::LSL,
10483 $src3$$constant & 0x1f);
10484 %}
10485
10486 ins_pipe(ialu_reg_reg_shift);
10487 %}
10488
10489 // This pattern is automatically generated from aarch64_ad.m4
10490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10491 // val ^ (-1 ^ (val << shift)) ==> eon
10492 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10493 iRegL src1, iRegL src2,
10494 immI src3, immL_M1 src4) %{
10495 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10496 ins_cost(1.9 * INSN_COST);
10497 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10498
10499 ins_encode %{
10500 __ eon(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 as_Register($src2$$reg),
10503 Assembler::LSL,
10504 $src3$$constant & 0x3f);
10505 %}
10506
10507 ins_pipe(ialu_reg_reg_shift);
10508 %}
10509
10510 // This pattern is automatically generated from aarch64_ad.m4
10511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10512 // val | (-1 ^ (val >>> shift)) ==> ornw
10513 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10514 iRegIorL2I src1, iRegIorL2I src2,
10515 immI src3, immI_M1 src4) %{
10516 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10517 ins_cost(1.9 * INSN_COST);
10518 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10519
10520 ins_encode %{
10521 __ ornw(as_Register($dst$$reg),
10522 as_Register($src1$$reg),
10523 as_Register($src2$$reg),
10524 Assembler::LSR,
10525 $src3$$constant & 0x1f);
10526 %}
10527
10528 ins_pipe(ialu_reg_reg_shift);
10529 %}
10530
10531 // This pattern is automatically generated from aarch64_ad.m4
10532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10533 // val | (-1 ^ (val >>> shift)) ==> orn
10534 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10535 iRegL src1, iRegL src2,
10536 immI src3, immL_M1 src4) %{
10537 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10538 ins_cost(1.9 * INSN_COST);
10539 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10540
10541 ins_encode %{
10542 __ orn(as_Register($dst$$reg),
10543 as_Register($src1$$reg),
10544 as_Register($src2$$reg),
10545 Assembler::LSR,
10546 $src3$$constant & 0x3f);
10547 %}
10548
10549 ins_pipe(ialu_reg_reg_shift);
10550 %}
10551
10552 // This pattern is automatically generated from aarch64_ad.m4
10553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10554 // val | (-1 ^ (val >> shift)) ==> ornw
10555 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10556 iRegIorL2I src1, iRegIorL2I src2,
10557 immI src3, immI_M1 src4) %{
10558 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10559 ins_cost(1.9 * INSN_COST);
10560 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10561
10562 ins_encode %{
10563 __ ornw(as_Register($dst$$reg),
10564 as_Register($src1$$reg),
10565 as_Register($src2$$reg),
10566 Assembler::ASR,
10567 $src3$$constant & 0x1f);
10568 %}
10569
10570 ins_pipe(ialu_reg_reg_shift);
10571 %}
10572
10573 // This pattern is automatically generated from aarch64_ad.m4
10574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10575 // val | (-1 ^ (val >> shift)) ==> orn
10576 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10577 iRegL src1, iRegL src2,
10578 immI src3, immL_M1 src4) %{
10579 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10580 ins_cost(1.9 * INSN_COST);
10581 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10582
10583 ins_encode %{
10584 __ orn(as_Register($dst$$reg),
10585 as_Register($src1$$reg),
10586 as_Register($src2$$reg),
10587 Assembler::ASR,
10588 $src3$$constant & 0x3f);
10589 %}
10590
10591 ins_pipe(ialu_reg_reg_shift);
10592 %}
10593
10594 // This pattern is automatically generated from aarch64_ad.m4
10595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10596 // val | (-1 ^ (val ror shift)) ==> ornw
10597 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10598 iRegIorL2I src1, iRegIorL2I src2,
10599 immI src3, immI_M1 src4) %{
10600 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10601 ins_cost(1.9 * INSN_COST);
10602 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10603
10604 ins_encode %{
10605 __ ornw(as_Register($dst$$reg),
10606 as_Register($src1$$reg),
10607 as_Register($src2$$reg),
10608 Assembler::ROR,
10609 $src3$$constant & 0x1f);
10610 %}
10611
10612 ins_pipe(ialu_reg_reg_shift);
10613 %}
10614
10615 // This pattern is automatically generated from aarch64_ad.m4
10616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10617 // val | (-1 ^ (val ror shift)) ==> orn
10618 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10619 iRegL src1, iRegL src2,
10620 immI src3, immL_M1 src4) %{
10621 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10622 ins_cost(1.9 * INSN_COST);
10623 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10624
10625 ins_encode %{
10626 __ orn(as_Register($dst$$reg),
10627 as_Register($src1$$reg),
10628 as_Register($src2$$reg),
10629 Assembler::ROR,
10630 $src3$$constant & 0x3f);
10631 %}
10632
10633 ins_pipe(ialu_reg_reg_shift);
10634 %}
10635
10636 // This pattern is automatically generated from aarch64_ad.m4
10637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10638 // val | (-1 ^ (val << shift)) ==> ornw
10639 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10640 iRegIorL2I src1, iRegIorL2I src2,
10641 immI src3, immI_M1 src4) %{
10642 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10643 ins_cost(1.9 * INSN_COST);
10644 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10645
10646 ins_encode %{
10647 __ ornw(as_Register($dst$$reg),
10648 as_Register($src1$$reg),
10649 as_Register($src2$$reg),
10650 Assembler::LSL,
10651 $src3$$constant & 0x1f);
10652 %}
10653
10654 ins_pipe(ialu_reg_reg_shift);
10655 %}
10656
10657 // This pattern is automatically generated from aarch64_ad.m4
10658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10659 // val | (-1 ^ (val << shift)) ==> orn
10660 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10661 iRegL src1, iRegL src2,
10662 immI src3, immL_M1 src4) %{
10663 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10664 ins_cost(1.9 * INSN_COST);
10665 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10666
10667 ins_encode %{
10668 __ orn(as_Register($dst$$reg),
10669 as_Register($src1$$reg),
10670 as_Register($src2$$reg),
10671 Assembler::LSL,
10672 $src3$$constant & 0x3f);
10673 %}
10674
10675 ins_pipe(ialu_reg_reg_shift);
10676 %}
10677
10678 // This pattern is automatically generated from aarch64_ad.m4
10679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10680 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10681 iRegIorL2I src1, iRegIorL2I src2,
10682 immI src3) %{
10683 match(Set dst (AndI src1 (URShiftI src2 src3)));
10684
10685 ins_cost(1.9 * INSN_COST);
10686 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10687
10688 ins_encode %{
10689 __ andw(as_Register($dst$$reg),
10690 as_Register($src1$$reg),
10691 as_Register($src2$$reg),
10692 Assembler::LSR,
10693 $src3$$constant & 0x1f);
10694 %}
10695
10696 ins_pipe(ialu_reg_reg_shift);
10697 %}
10698
10699 // This pattern is automatically generated from aarch64_ad.m4
10700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10701 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10702 iRegL src1, iRegL src2,
10703 immI src3) %{
10704 match(Set dst (AndL src1 (URShiftL src2 src3)));
10705
10706 ins_cost(1.9 * INSN_COST);
10707 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10708
10709 ins_encode %{
10710 __ andr(as_Register($dst$$reg),
10711 as_Register($src1$$reg),
10712 as_Register($src2$$reg),
10713 Assembler::LSR,
10714 $src3$$constant & 0x3f);
10715 %}
10716
10717 ins_pipe(ialu_reg_reg_shift);
10718 %}
10719
10720 // This pattern is automatically generated from aarch64_ad.m4
10721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10722 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10723 iRegIorL2I src1, iRegIorL2I src2,
10724 immI src3) %{
10725 match(Set dst (AndI src1 (RShiftI src2 src3)));
10726
10727 ins_cost(1.9 * INSN_COST);
10728 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10729
10730 ins_encode %{
10731 __ andw(as_Register($dst$$reg),
10732 as_Register($src1$$reg),
10733 as_Register($src2$$reg),
10734 Assembler::ASR,
10735 $src3$$constant & 0x1f);
10736 %}
10737
10738 ins_pipe(ialu_reg_reg_shift);
10739 %}
10740
10741 // This pattern is automatically generated from aarch64_ad.m4
10742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10743 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10744 iRegL src1, iRegL src2,
10745 immI src3) %{
10746 match(Set dst (AndL src1 (RShiftL src2 src3)));
10747
10748 ins_cost(1.9 * INSN_COST);
10749 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10750
10751 ins_encode %{
10752 __ andr(as_Register($dst$$reg),
10753 as_Register($src1$$reg),
10754 as_Register($src2$$reg),
10755 Assembler::ASR,
10756 $src3$$constant & 0x3f);
10757 %}
10758
10759 ins_pipe(ialu_reg_reg_shift);
10760 %}
10761
10762 // This pattern is automatically generated from aarch64_ad.m4
10763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10764 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10765 iRegIorL2I src1, iRegIorL2I src2,
10766 immI src3) %{
10767 match(Set dst (AndI src1 (LShiftI src2 src3)));
10768
10769 ins_cost(1.9 * INSN_COST);
10770 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10771
10772 ins_encode %{
10773 __ andw(as_Register($dst$$reg),
10774 as_Register($src1$$reg),
10775 as_Register($src2$$reg),
10776 Assembler::LSL,
10777 $src3$$constant & 0x1f);
10778 %}
10779
10780 ins_pipe(ialu_reg_reg_shift);
10781 %}
10782
10783 // This pattern is automatically generated from aarch64_ad.m4
10784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10785 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10786 iRegL src1, iRegL src2,
10787 immI src3) %{
10788 match(Set dst (AndL src1 (LShiftL src2 src3)));
10789
10790 ins_cost(1.9 * INSN_COST);
10791 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10792
10793 ins_encode %{
10794 __ andr(as_Register($dst$$reg),
10795 as_Register($src1$$reg),
10796 as_Register($src2$$reg),
10797 Assembler::LSL,
10798 $src3$$constant & 0x3f);
10799 %}
10800
10801 ins_pipe(ialu_reg_reg_shift);
10802 %}
10803
10804 // This pattern is automatically generated from aarch64_ad.m4
10805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10806 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10807 iRegIorL2I src1, iRegIorL2I src2,
10808 immI src3) %{
10809 match(Set dst (AndI src1 (RotateRight src2 src3)));
10810
10811 ins_cost(1.9 * INSN_COST);
10812 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10813
10814 ins_encode %{
10815 __ andw(as_Register($dst$$reg),
10816 as_Register($src1$$reg),
10817 as_Register($src2$$reg),
10818 Assembler::ROR,
10819 $src3$$constant & 0x1f);
10820 %}
10821
10822 ins_pipe(ialu_reg_reg_shift);
10823 %}
10824
10825 // This pattern is automatically generated from aarch64_ad.m4
10826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10827 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10828 iRegL src1, iRegL src2,
10829 immI src3) %{
10830 match(Set dst (AndL src1 (RotateRight src2 src3)));
10831
10832 ins_cost(1.9 * INSN_COST);
10833 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10834
10835 ins_encode %{
10836 __ andr(as_Register($dst$$reg),
10837 as_Register($src1$$reg),
10838 as_Register($src2$$reg),
10839 Assembler::ROR,
10840 $src3$$constant & 0x3f);
10841 %}
10842
10843 ins_pipe(ialu_reg_reg_shift);
10844 %}
10845
10846 // This pattern is automatically generated from aarch64_ad.m4
10847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10848 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10849 iRegIorL2I src1, iRegIorL2I src2,
10850 immI src3) %{
10851 match(Set dst (XorI src1 (URShiftI src2 src3)));
10852
10853 ins_cost(1.9 * INSN_COST);
10854 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10855
10856 ins_encode %{
10857 __ eorw(as_Register($dst$$reg),
10858 as_Register($src1$$reg),
10859 as_Register($src2$$reg),
10860 Assembler::LSR,
10861 $src3$$constant & 0x1f);
10862 %}
10863
10864 ins_pipe(ialu_reg_reg_shift);
10865 %}
10866
10867 // This pattern is automatically generated from aarch64_ad.m4
10868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10869 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10870 iRegL src1, iRegL src2,
10871 immI src3) %{
10872 match(Set dst (XorL src1 (URShiftL src2 src3)));
10873
10874 ins_cost(1.9 * INSN_COST);
10875 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10876
10877 ins_encode %{
10878 __ eor(as_Register($dst$$reg),
10879 as_Register($src1$$reg),
10880 as_Register($src2$$reg),
10881 Assembler::LSR,
10882 $src3$$constant & 0x3f);
10883 %}
10884
10885 ins_pipe(ialu_reg_reg_shift);
10886 %}
10887
10888 // This pattern is automatically generated from aarch64_ad.m4
10889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10890 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10891 iRegIorL2I src1, iRegIorL2I src2,
10892 immI src3) %{
10893 match(Set dst (XorI src1 (RShiftI src2 src3)));
10894
10895 ins_cost(1.9 * INSN_COST);
10896 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10897
10898 ins_encode %{
10899 __ eorw(as_Register($dst$$reg),
10900 as_Register($src1$$reg),
10901 as_Register($src2$$reg),
10902 Assembler::ASR,
10903 $src3$$constant & 0x1f);
10904 %}
10905
10906 ins_pipe(ialu_reg_reg_shift);
10907 %}
10908
10909 // This pattern is automatically generated from aarch64_ad.m4
10910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10911 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10912 iRegL src1, iRegL src2,
10913 immI src3) %{
10914 match(Set dst (XorL src1 (RShiftL src2 src3)));
10915
10916 ins_cost(1.9 * INSN_COST);
10917 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10918
10919 ins_encode %{
10920 __ eor(as_Register($dst$$reg),
10921 as_Register($src1$$reg),
10922 as_Register($src2$$reg),
10923 Assembler::ASR,
10924 $src3$$constant & 0x3f);
10925 %}
10926
10927 ins_pipe(ialu_reg_reg_shift);
10928 %}
10929
10930 // This pattern is automatically generated from aarch64_ad.m4
10931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10932 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10933 iRegIorL2I src1, iRegIorL2I src2,
10934 immI src3) %{
10935 match(Set dst (XorI src1 (LShiftI src2 src3)));
10936
10937 ins_cost(1.9 * INSN_COST);
10938 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10939
10940 ins_encode %{
10941 __ eorw(as_Register($dst$$reg),
10942 as_Register($src1$$reg),
10943 as_Register($src2$$reg),
10944 Assembler::LSL,
10945 $src3$$constant & 0x1f);
10946 %}
10947
10948 ins_pipe(ialu_reg_reg_shift);
10949 %}
10950
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10954 iRegL src1, iRegL src2,
10955 immI src3) %{
10956 match(Set dst (XorL src1 (LShiftL src2 src3)));
10957
10958 ins_cost(1.9 * INSN_COST);
10959 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10960
10961 ins_encode %{
10962 __ eor(as_Register($dst$$reg),
10963 as_Register($src1$$reg),
10964 as_Register($src2$$reg),
10965 Assembler::LSL,
10966 $src3$$constant & 0x3f);
10967 %}
10968
10969 ins_pipe(ialu_reg_reg_shift);
10970 %}
10971
10972 // This pattern is automatically generated from aarch64_ad.m4
10973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10974 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10975 iRegIorL2I src1, iRegIorL2I src2,
10976 immI src3) %{
10977 match(Set dst (XorI src1 (RotateRight src2 src3)));
10978
10979 ins_cost(1.9 * INSN_COST);
10980 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10981
10982 ins_encode %{
10983 __ eorw(as_Register($dst$$reg),
10984 as_Register($src1$$reg),
10985 as_Register($src2$$reg),
10986 Assembler::ROR,
10987 $src3$$constant & 0x1f);
10988 %}
10989
10990 ins_pipe(ialu_reg_reg_shift);
10991 %}
10992
10993 // This pattern is automatically generated from aarch64_ad.m4
10994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10995 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10996 iRegL src1, iRegL src2,
10997 immI src3) %{
10998 match(Set dst (XorL src1 (RotateRight src2 src3)));
10999
11000 ins_cost(1.9 * INSN_COST);
11001 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11002
11003 ins_encode %{
11004 __ eor(as_Register($dst$$reg),
11005 as_Register($src1$$reg),
11006 as_Register($src2$$reg),
11007 Assembler::ROR,
11008 $src3$$constant & 0x3f);
11009 %}
11010
11011 ins_pipe(ialu_reg_reg_shift);
11012 %}
11013
11014 // This pattern is automatically generated from aarch64_ad.m4
11015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11016 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11017 iRegIorL2I src1, iRegIorL2I src2,
11018 immI src3) %{
11019 match(Set dst (OrI src1 (URShiftI src2 src3)));
11020
11021 ins_cost(1.9 * INSN_COST);
11022 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11023
11024 ins_encode %{
11025 __ orrw(as_Register($dst$$reg),
11026 as_Register($src1$$reg),
11027 as_Register($src2$$reg),
11028 Assembler::LSR,
11029 $src3$$constant & 0x1f);
11030 %}
11031
11032 ins_pipe(ialu_reg_reg_shift);
11033 %}
11034
11035 // This pattern is automatically generated from aarch64_ad.m4
11036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11037 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11038 iRegL src1, iRegL src2,
11039 immI src3) %{
11040 match(Set dst (OrL src1 (URShiftL src2 src3)));
11041
11042 ins_cost(1.9 * INSN_COST);
11043 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11044
11045 ins_encode %{
11046 __ orr(as_Register($dst$$reg),
11047 as_Register($src1$$reg),
11048 as_Register($src2$$reg),
11049 Assembler::LSR,
11050 $src3$$constant & 0x3f);
11051 %}
11052
11053 ins_pipe(ialu_reg_reg_shift);
11054 %}
11055
11056 // This pattern is automatically generated from aarch64_ad.m4
11057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11058 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11059 iRegIorL2I src1, iRegIorL2I src2,
11060 immI src3) %{
11061 match(Set dst (OrI src1 (RShiftI src2 src3)));
11062
11063 ins_cost(1.9 * INSN_COST);
11064 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11065
11066 ins_encode %{
11067 __ orrw(as_Register($dst$$reg),
11068 as_Register($src1$$reg),
11069 as_Register($src2$$reg),
11070 Assembler::ASR,
11071 $src3$$constant & 0x1f);
11072 %}
11073
11074 ins_pipe(ialu_reg_reg_shift);
11075 %}
11076
11077 // This pattern is automatically generated from aarch64_ad.m4
11078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11079 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11080 iRegL src1, iRegL src2,
11081 immI src3) %{
11082 match(Set dst (OrL src1 (RShiftL src2 src3)));
11083
11084 ins_cost(1.9 * INSN_COST);
11085 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11086
11087 ins_encode %{
11088 __ orr(as_Register($dst$$reg),
11089 as_Register($src1$$reg),
11090 as_Register($src2$$reg),
11091 Assembler::ASR,
11092 $src3$$constant & 0x3f);
11093 %}
11094
11095 ins_pipe(ialu_reg_reg_shift);
11096 %}
11097
11098 // This pattern is automatically generated from aarch64_ad.m4
11099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11100 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11101 iRegIorL2I src1, iRegIorL2I src2,
11102 immI src3) %{
11103 match(Set dst (OrI src1 (LShiftI src2 src3)));
11104
11105 ins_cost(1.9 * INSN_COST);
11106 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11107
11108 ins_encode %{
11109 __ orrw(as_Register($dst$$reg),
11110 as_Register($src1$$reg),
11111 as_Register($src2$$reg),
11112 Assembler::LSL,
11113 $src3$$constant & 0x1f);
11114 %}
11115
11116 ins_pipe(ialu_reg_reg_shift);
11117 %}
11118
11119 // This pattern is automatically generated from aarch64_ad.m4
11120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11121 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11122 iRegL src1, iRegL src2,
11123 immI src3) %{
11124 match(Set dst (OrL src1 (LShiftL src2 src3)));
11125
11126 ins_cost(1.9 * INSN_COST);
11127 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11128
11129 ins_encode %{
11130 __ orr(as_Register($dst$$reg),
11131 as_Register($src1$$reg),
11132 as_Register($src2$$reg),
11133 Assembler::LSL,
11134 $src3$$constant & 0x3f);
11135 %}
11136
11137 ins_pipe(ialu_reg_reg_shift);
11138 %}
11139
11140 // This pattern is automatically generated from aarch64_ad.m4
11141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11142 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11143 iRegIorL2I src1, iRegIorL2I src2,
11144 immI src3) %{
11145 match(Set dst (OrI src1 (RotateRight src2 src3)));
11146
11147 ins_cost(1.9 * INSN_COST);
11148 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11149
11150 ins_encode %{
11151 __ orrw(as_Register($dst$$reg),
11152 as_Register($src1$$reg),
11153 as_Register($src2$$reg),
11154 Assembler::ROR,
11155 $src3$$constant & 0x1f);
11156 %}
11157
11158 ins_pipe(ialu_reg_reg_shift);
11159 %}
11160
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11164 iRegL src1, iRegL src2,
11165 immI src3) %{
11166 match(Set dst (OrL src1 (RotateRight src2 src3)));
11167
11168 ins_cost(1.9 * INSN_COST);
11169 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11170
11171 ins_encode %{
11172 __ orr(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg),
11175 Assembler::ROR,
11176 $src3$$constant & 0x3f);
11177 %}
11178
11179 ins_pipe(ialu_reg_reg_shift);
11180 %}
11181
11182 // This pattern is automatically generated from aarch64_ad.m4
11183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11184 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11185 iRegIorL2I src1, iRegIorL2I src2,
11186 immI src3) %{
11187 match(Set dst (AddI src1 (URShiftI src2 src3)));
11188
11189 ins_cost(1.9 * INSN_COST);
11190 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11191
11192 ins_encode %{
11193 __ addw(as_Register($dst$$reg),
11194 as_Register($src1$$reg),
11195 as_Register($src2$$reg),
11196 Assembler::LSR,
11197 $src3$$constant & 0x1f);
11198 %}
11199
11200 ins_pipe(ialu_reg_reg_shift);
11201 %}
11202
11203 // This pattern is automatically generated from aarch64_ad.m4
11204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11205 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11206 iRegL src1, iRegL src2,
11207 immI src3) %{
11208 match(Set dst (AddL src1 (URShiftL src2 src3)));
11209
11210 ins_cost(1.9 * INSN_COST);
11211 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11212
11213 ins_encode %{
11214 __ add(as_Register($dst$$reg),
11215 as_Register($src1$$reg),
11216 as_Register($src2$$reg),
11217 Assembler::LSR,
11218 $src3$$constant & 0x3f);
11219 %}
11220
11221 ins_pipe(ialu_reg_reg_shift);
11222 %}
11223
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11227 iRegIorL2I src1, iRegIorL2I src2,
11228 immI src3) %{
11229 match(Set dst (AddI src1 (RShiftI src2 src3)));
11230
11231 ins_cost(1.9 * INSN_COST);
11232 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11233
11234 ins_encode %{
11235 __ addw(as_Register($dst$$reg),
11236 as_Register($src1$$reg),
11237 as_Register($src2$$reg),
11238 Assembler::ASR,
11239 $src3$$constant & 0x1f);
11240 %}
11241
11242 ins_pipe(ialu_reg_reg_shift);
11243 %}
11244
11245 // This pattern is automatically generated from aarch64_ad.m4
11246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11247 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11248 iRegL src1, iRegL src2,
11249 immI src3) %{
11250 match(Set dst (AddL src1 (RShiftL src2 src3)));
11251
11252 ins_cost(1.9 * INSN_COST);
11253 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11254
11255 ins_encode %{
11256 __ add(as_Register($dst$$reg),
11257 as_Register($src1$$reg),
11258 as_Register($src2$$reg),
11259 Assembler::ASR,
11260 $src3$$constant & 0x3f);
11261 %}
11262
11263 ins_pipe(ialu_reg_reg_shift);
11264 %}
11265
11266 // This pattern is automatically generated from aarch64_ad.m4
11267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11268 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11269 iRegIorL2I src1, iRegIorL2I src2,
11270 immI src3) %{
11271 match(Set dst (AddI src1 (LShiftI src2 src3)));
11272
11273 ins_cost(1.9 * INSN_COST);
11274 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11275
11276 ins_encode %{
11277 __ addw(as_Register($dst$$reg),
11278 as_Register($src1$$reg),
11279 as_Register($src2$$reg),
11280 Assembler::LSL,
11281 $src3$$constant & 0x1f);
11282 %}
11283
11284 ins_pipe(ialu_reg_reg_shift);
11285 %}
11286
11287 // This pattern is automatically generated from aarch64_ad.m4
11288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11289 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11290 iRegL src1, iRegL src2,
11291 immI src3) %{
11292 match(Set dst (AddL src1 (LShiftL src2 src3)));
11293
11294 ins_cost(1.9 * INSN_COST);
11295 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11296
11297 ins_encode %{
11298 __ add(as_Register($dst$$reg),
11299 as_Register($src1$$reg),
11300 as_Register($src2$$reg),
11301 Assembler::LSL,
11302 $src3$$constant & 0x3f);
11303 %}
11304
11305 ins_pipe(ialu_reg_reg_shift);
11306 %}
11307
11308 // This pattern is automatically generated from aarch64_ad.m4
11309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11310 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11311 iRegIorL2I src1, iRegIorL2I src2,
11312 immI src3) %{
11313 match(Set dst (SubI src1 (URShiftI src2 src3)));
11314
11315 ins_cost(1.9 * INSN_COST);
11316 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11317
11318 ins_encode %{
11319 __ subw(as_Register($dst$$reg),
11320 as_Register($src1$$reg),
11321 as_Register($src2$$reg),
11322 Assembler::LSR,
11323 $src3$$constant & 0x1f);
11324 %}
11325
11326 ins_pipe(ialu_reg_reg_shift);
11327 %}
11328
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11332 iRegL src1, iRegL src2,
11333 immI src3) %{
11334 match(Set dst (SubL src1 (URShiftL src2 src3)));
11335
11336 ins_cost(1.9 * INSN_COST);
11337 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11338
11339 ins_encode %{
11340 __ sub(as_Register($dst$$reg),
11341 as_Register($src1$$reg),
11342 as_Register($src2$$reg),
11343 Assembler::LSR,
11344 $src3$$constant & 0x3f);
11345 %}
11346
11347 ins_pipe(ialu_reg_reg_shift);
11348 %}
11349
11350 // This pattern is automatically generated from aarch64_ad.m4
11351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11352 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11353 iRegIorL2I src1, iRegIorL2I src2,
11354 immI src3) %{
11355 match(Set dst (SubI src1 (RShiftI src2 src3)));
11356
11357 ins_cost(1.9 * INSN_COST);
11358 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11359
11360 ins_encode %{
11361 __ subw(as_Register($dst$$reg),
11362 as_Register($src1$$reg),
11363 as_Register($src2$$reg),
11364 Assembler::ASR,
11365 $src3$$constant & 0x1f);
11366 %}
11367
11368 ins_pipe(ialu_reg_reg_shift);
11369 %}
11370
11371 // This pattern is automatically generated from aarch64_ad.m4
11372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11373 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11374 iRegL src1, iRegL src2,
11375 immI src3) %{
11376 match(Set dst (SubL src1 (RShiftL src2 src3)));
11377
11378 ins_cost(1.9 * INSN_COST);
11379 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11380
11381 ins_encode %{
11382 __ sub(as_Register($dst$$reg),
11383 as_Register($src1$$reg),
11384 as_Register($src2$$reg),
11385 Assembler::ASR,
11386 $src3$$constant & 0x3f);
11387 %}
11388
11389 ins_pipe(ialu_reg_reg_shift);
11390 %}
11391
11392 // This pattern is automatically generated from aarch64_ad.m4
11393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11394 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11395 iRegIorL2I src1, iRegIorL2I src2,
11396 immI src3) %{
11397 match(Set dst (SubI src1 (LShiftI src2 src3)));
11398
11399 ins_cost(1.9 * INSN_COST);
11400 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11401
11402 ins_encode %{
11403 __ subw(as_Register($dst$$reg),
11404 as_Register($src1$$reg),
11405 as_Register($src2$$reg),
11406 Assembler::LSL,
11407 $src3$$constant & 0x1f);
11408 %}
11409
11410 ins_pipe(ialu_reg_reg_shift);
11411 %}
11412
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11416 iRegL src1, iRegL src2,
11417 immI src3) %{
11418 match(Set dst (SubL src1 (LShiftL src2 src3)));
11419
11420 ins_cost(1.9 * INSN_COST);
11421 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11422
11423 ins_encode %{
11424 __ sub(as_Register($dst$$reg),
11425 as_Register($src1$$reg),
11426 as_Register($src2$$reg),
11427 Assembler::LSL,
11428 $src3$$constant & 0x3f);
11429 %}
11430
11431 ins_pipe(ialu_reg_reg_shift);
11432 %}
11433
11434 // This pattern is automatically generated from aarch64_ad.m4
11435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11436
11437 // Shift Left followed by Shift Right.
11438 // This idiom is used by the compiler for the i2b bytecode etc.
11439 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11440 %{
11441 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11442 ins_cost(INSN_COST * 2);
11443 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11444 ins_encode %{
11445 int lshift = $lshift_count$$constant & 63;
11446 int rshift = $rshift_count$$constant & 63;
11447 int s = 63 - lshift;
11448 int r = (rshift - lshift) & 63;
11449 __ sbfm(as_Register($dst$$reg),
11450 as_Register($src$$reg),
11451 r, s);
11452 %}
11453
11454 ins_pipe(ialu_reg_shift);
11455 %}
11456
11457 // This pattern is automatically generated from aarch64_ad.m4
11458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11459
11460 // Shift Left followed by Shift Right.
11461 // This idiom is used by the compiler for the i2b bytecode etc.
11462 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11463 %{
11464 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11465 ins_cost(INSN_COST * 2);
11466 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11467 ins_encode %{
11468 int lshift = $lshift_count$$constant & 31;
11469 int rshift = $rshift_count$$constant & 31;
11470 int s = 31 - lshift;
11471 int r = (rshift - lshift) & 31;
11472 __ sbfmw(as_Register($dst$$reg),
11473 as_Register($src$$reg),
11474 r, s);
11475 %}
11476
11477 ins_pipe(ialu_reg_shift);
11478 %}
11479
11480 // This pattern is automatically generated from aarch64_ad.m4
11481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11482
11483 // Shift Left followed by Shift Right.
11484 // This idiom is used by the compiler for the i2b bytecode etc.
11485 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11486 %{
11487 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11488 ins_cost(INSN_COST * 2);
11489 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11490 ins_encode %{
11491 int lshift = $lshift_count$$constant & 63;
11492 int rshift = $rshift_count$$constant & 63;
11493 int s = 63 - lshift;
11494 int r = (rshift - lshift) & 63;
11495 __ ubfm(as_Register($dst$$reg),
11496 as_Register($src$$reg),
11497 r, s);
11498 %}
11499
11500 ins_pipe(ialu_reg_shift);
11501 %}
11502
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505
11506 // Shift Left followed by Shift Right.
11507 // This idiom is used by the compiler for the i2b bytecode etc.
11508 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11509 %{
11510 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11511 ins_cost(INSN_COST * 2);
11512 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11513 ins_encode %{
11514 int lshift = $lshift_count$$constant & 31;
11515 int rshift = $rshift_count$$constant & 31;
11516 int s = 31 - lshift;
11517 int r = (rshift - lshift) & 31;
11518 __ ubfmw(as_Register($dst$$reg),
11519 as_Register($src$$reg),
11520 r, s);
11521 %}
11522
11523 ins_pipe(ialu_reg_shift);
11524 %}
11525
11526 // Bitfield extract with shift & mask
11527
11528 // This pattern is automatically generated from aarch64_ad.m4
11529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11530 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11531 %{
11532 match(Set dst (AndI (URShiftI src rshift) mask));
11533 // Make sure we are not going to exceed what ubfxw can do.
11534 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11535
11536 ins_cost(INSN_COST);
11537 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11538 ins_encode %{
11539 int rshift = $rshift$$constant & 31;
11540 intptr_t mask = $mask$$constant;
11541 int width = exact_log2(mask+1);
11542 __ ubfxw(as_Register($dst$$reg),
11543 as_Register($src$$reg), rshift, width);
11544 %}
11545 ins_pipe(ialu_reg_shift);
11546 %}
11547
11548 // This pattern is automatically generated from aarch64_ad.m4
11549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11550 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11551 %{
11552 match(Set dst (AndL (URShiftL src rshift) mask));
11553 // Make sure we are not going to exceed what ubfx can do.
11554 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11555
11556 ins_cost(INSN_COST);
11557 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11558 ins_encode %{
11559 int rshift = $rshift$$constant & 63;
11560 intptr_t mask = $mask$$constant;
11561 int width = exact_log2_long(mask+1);
11562 __ ubfx(as_Register($dst$$reg),
11563 as_Register($src$$reg), rshift, width);
11564 %}
11565 ins_pipe(ialu_reg_shift);
11566 %}
11567
11568
11569 // This pattern is automatically generated from aarch64_ad.m4
11570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11571
11572 // We can use ubfx when extending an And with a mask when we know mask
11573 // is positive. We know that because immI_bitmask guarantees it.
11574 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11575 %{
11576 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11577 // Make sure we are not going to exceed what ubfxw can do.
11578 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11579
11580 ins_cost(INSN_COST * 2);
11581 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11582 ins_encode %{
11583 int rshift = $rshift$$constant & 31;
11584 intptr_t mask = $mask$$constant;
11585 int width = exact_log2(mask+1);
11586 __ ubfx(as_Register($dst$$reg),
11587 as_Register($src$$reg), rshift, width);
11588 %}
11589 ins_pipe(ialu_reg_shift);
11590 %}
11591
11592
11593 // This pattern is automatically generated from aarch64_ad.m4
11594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11595
11596 // We can use ubfiz when masking by a positive number and then left shifting the result.
11597 // We know that the mask is positive because immI_bitmask guarantees it.
11598 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11599 %{
11600 match(Set dst (LShiftI (AndI src mask) lshift));
11601 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11602
11603 ins_cost(INSN_COST);
11604 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11605 ins_encode %{
11606 int lshift = $lshift$$constant & 31;
11607 intptr_t mask = $mask$$constant;
11608 int width = exact_log2(mask+1);
11609 __ ubfizw(as_Register($dst$$reg),
11610 as_Register($src$$reg), lshift, width);
11611 %}
11612 ins_pipe(ialu_reg_shift);
11613 %}
11614
11615 // This pattern is automatically generated from aarch64_ad.m4
11616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11617
11618 // We can use ubfiz when masking by a positive number and then left shifting the result.
11619 // We know that the mask is positive because immL_bitmask guarantees it.
11620 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11621 %{
11622 match(Set dst (LShiftL (AndL src mask) lshift));
11623 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11624
11625 ins_cost(INSN_COST);
11626 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11627 ins_encode %{
11628 int lshift = $lshift$$constant & 63;
11629 intptr_t mask = $mask$$constant;
11630 int width = exact_log2_long(mask+1);
11631 __ ubfiz(as_Register($dst$$reg),
11632 as_Register($src$$reg), lshift, width);
11633 %}
11634 ins_pipe(ialu_reg_shift);
11635 %}
11636
11637 // This pattern is automatically generated from aarch64_ad.m4
11638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11639
11640 // We can use ubfiz when masking by a positive number and then left shifting the result.
11641 // We know that the mask is positive because immI_bitmask guarantees it.
11642 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11643 %{
11644 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11645 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11646
11647 ins_cost(INSN_COST);
11648 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11649 ins_encode %{
11650 int lshift = $lshift$$constant & 31;
11651 intptr_t mask = $mask$$constant;
11652 int width = exact_log2(mask+1);
11653 __ ubfizw(as_Register($dst$$reg),
11654 as_Register($src$$reg), lshift, width);
11655 %}
11656 ins_pipe(ialu_reg_shift);
11657 %}
11658
11659 // This pattern is automatically generated from aarch64_ad.m4
11660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11661
11662 // We can use ubfiz when masking by a positive number and then left shifting the result.
11663 // We know that the mask is positive because immL_bitmask guarantees it.
11664 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11665 %{
11666 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11667 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11668
11669 ins_cost(INSN_COST);
11670 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11671 ins_encode %{
11672 int lshift = $lshift$$constant & 63;
11673 intptr_t mask = $mask$$constant;
11674 int width = exact_log2_long(mask+1);
11675 __ ubfiz(as_Register($dst$$reg),
11676 as_Register($src$$reg), lshift, width);
11677 %}
11678 ins_pipe(ialu_reg_shift);
11679 %}
11680
11681
11682 // This pattern is automatically generated from aarch64_ad.m4
11683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11684
11685 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11686 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11687 %{
11688 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11689 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11690
11691 ins_cost(INSN_COST);
11692 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11693 ins_encode %{
11694 int lshift = $lshift$$constant & 63;
11695 intptr_t mask = $mask$$constant;
11696 int width = exact_log2(mask+1);
11697 __ ubfiz(as_Register($dst$$reg),
11698 as_Register($src$$reg), lshift, width);
11699 %}
11700 ins_pipe(ialu_reg_shift);
11701 %}
11702
11703 // This pattern is automatically generated from aarch64_ad.m4
11704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11705
11706 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11707 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11708 %{
11709 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11710 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11711
11712 ins_cost(INSN_COST);
11713 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11714 ins_encode %{
11715 int lshift = $lshift$$constant & 31;
11716 intptr_t mask = $mask$$constant;
11717 int width = exact_log2(mask+1);
11718 __ ubfiz(as_Register($dst$$reg),
11719 as_Register($src$$reg), lshift, width);
11720 %}
11721 ins_pipe(ialu_reg_shift);
11722 %}
11723
11724 // This pattern is automatically generated from aarch64_ad.m4
11725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11726
11727 // Can skip int2long conversions after AND with small bitmask
11728 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11729 %{
11730 match(Set dst (ConvI2L (AndI src msk)));
11731 ins_cost(INSN_COST);
11732 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11733 ins_encode %{
11734 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11735 %}
11736 ins_pipe(ialu_reg_shift);
11737 %}
11738
11739
11740 // Rotations
11741
11742 // This pattern is automatically generated from aarch64_ad.m4
11743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11744 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11745 %{
11746 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11747 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11748
11749 ins_cost(INSN_COST);
11750 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11751
11752 ins_encode %{
11753 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11754 $rshift$$constant & 63);
11755 %}
11756 ins_pipe(ialu_reg_reg_extr);
11757 %}
11758
11759
11760 // This pattern is automatically generated from aarch64_ad.m4
11761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11762 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11763 %{
11764 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11765 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11766
11767 ins_cost(INSN_COST);
11768 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11769
11770 ins_encode %{
11771 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11772 $rshift$$constant & 31);
11773 %}
11774 ins_pipe(ialu_reg_reg_extr);
11775 %}
11776
11777
11778 // This pattern is automatically generated from aarch64_ad.m4
11779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11780 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11781 %{
11782 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11783 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11784
11785 ins_cost(INSN_COST);
11786 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11787
11788 ins_encode %{
11789 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11790 $rshift$$constant & 63);
11791 %}
11792 ins_pipe(ialu_reg_reg_extr);
11793 %}
11794
11795
11796 // This pattern is automatically generated from aarch64_ad.m4
11797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11798 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11799 %{
11800 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11801 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11802
11803 ins_cost(INSN_COST);
11804 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11805
11806 ins_encode %{
11807 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11808 $rshift$$constant & 31);
11809 %}
11810 ins_pipe(ialu_reg_reg_extr);
11811 %}
11812
11813 // This pattern is automatically generated from aarch64_ad.m4
11814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11815 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11816 %{
11817 match(Set dst (RotateRight src shift));
11818
11819 ins_cost(INSN_COST);
11820 format %{ "ror $dst, $src, $shift" %}
11821
11822 ins_encode %{
11823 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11824 $shift$$constant & 0x1f);
11825 %}
11826 ins_pipe(ialu_reg_reg_vshift);
11827 %}
11828
11829 // This pattern is automatically generated from aarch64_ad.m4
11830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11831 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11832 %{
11833 match(Set dst (RotateRight src shift));
11834
11835 ins_cost(INSN_COST);
11836 format %{ "ror $dst, $src, $shift" %}
11837
11838 ins_encode %{
11839 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11840 $shift$$constant & 0x3f);
11841 %}
11842 ins_pipe(ialu_reg_reg_vshift);
11843 %}
11844
11845 // This pattern is automatically generated from aarch64_ad.m4
11846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11847 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11848 %{
11849 match(Set dst (RotateRight src shift));
11850
11851 ins_cost(INSN_COST);
11852 format %{ "ror $dst, $src, $shift" %}
11853
11854 ins_encode %{
11855 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11856 %}
11857 ins_pipe(ialu_reg_reg_vshift);
11858 %}
11859
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11863 %{
11864 match(Set dst (RotateRight src shift));
11865
11866 ins_cost(INSN_COST);
11867 format %{ "ror $dst, $src, $shift" %}
11868
11869 ins_encode %{
11870 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11871 %}
11872 ins_pipe(ialu_reg_reg_vshift);
11873 %}
11874
11875 // This pattern is automatically generated from aarch64_ad.m4
11876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11877 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11878 %{
11879 match(Set dst (RotateLeft src shift));
11880
11881 ins_cost(INSN_COST);
11882 format %{ "rol $dst, $src, $shift" %}
11883
11884 ins_encode %{
11885 __ subw(rscratch1, zr, as_Register($shift$$reg));
11886 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11887 %}
11888 ins_pipe(ialu_reg_reg_vshift);
11889 %}
11890
11891 // This pattern is automatically generated from aarch64_ad.m4
11892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11893 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11894 %{
11895 match(Set dst (RotateLeft src shift));
11896
11897 ins_cost(INSN_COST);
11898 format %{ "rol $dst, $src, $shift" %}
11899
11900 ins_encode %{
11901 __ subw(rscratch1, zr, as_Register($shift$$reg));
11902 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11903 %}
11904 ins_pipe(ialu_reg_reg_vshift);
11905 %}
11906
11907
11908 // Add/subtract (extended)
11909
11910 // This pattern is automatically generated from aarch64_ad.m4
11911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11912 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11913 %{
11914 match(Set dst (AddL src1 (ConvI2L src2)));
11915 ins_cost(INSN_COST);
11916 format %{ "add $dst, $src1, $src2, sxtw" %}
11917
11918 ins_encode %{
11919 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11920 as_Register($src2$$reg), ext::sxtw);
11921 %}
11922 ins_pipe(ialu_reg_reg);
11923 %}
11924
11925 // This pattern is automatically generated from aarch64_ad.m4
11926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11927 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11928 %{
11929 match(Set dst (SubL src1 (ConvI2L src2)));
11930 ins_cost(INSN_COST);
11931 format %{ "sub $dst, $src1, $src2, sxtw" %}
11932
11933 ins_encode %{
11934 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11935 as_Register($src2$$reg), ext::sxtw);
11936 %}
11937 ins_pipe(ialu_reg_reg);
11938 %}
11939
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11943 %{
11944 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11945 ins_cost(INSN_COST);
11946 format %{ "add $dst, $src1, $src2, sxth" %}
11947
11948 ins_encode %{
11949 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11950 as_Register($src2$$reg), ext::sxth);
11951 %}
11952 ins_pipe(ialu_reg_reg);
11953 %}
11954
11955 // This pattern is automatically generated from aarch64_ad.m4
11956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11957 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11958 %{
11959 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11960 ins_cost(INSN_COST);
11961 format %{ "add $dst, $src1, $src2, sxtb" %}
11962
11963 ins_encode %{
11964 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11965 as_Register($src2$$reg), ext::sxtb);
11966 %}
11967 ins_pipe(ialu_reg_reg);
11968 %}
11969
11970 // This pattern is automatically generated from aarch64_ad.m4
11971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11972 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11973 %{
11974 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11975 ins_cost(INSN_COST);
11976 format %{ "add $dst, $src1, $src2, uxtb" %}
11977
11978 ins_encode %{
11979 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11980 as_Register($src2$$reg), ext::uxtb);
11981 %}
11982 ins_pipe(ialu_reg_reg);
11983 %}
11984
11985 // This pattern is automatically generated from aarch64_ad.m4
11986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11987 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11988 %{
11989 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11990 ins_cost(INSN_COST);
11991 format %{ "add $dst, $src1, $src2, sxth" %}
11992
11993 ins_encode %{
11994 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11995 as_Register($src2$$reg), ext::sxth);
11996 %}
11997 ins_pipe(ialu_reg_reg);
11998 %}
11999
12000 // This pattern is automatically generated from aarch64_ad.m4
12001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12002 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12003 %{
12004 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12005 ins_cost(INSN_COST);
12006 format %{ "add $dst, $src1, $src2, sxtw" %}
12007
12008 ins_encode %{
12009 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12010 as_Register($src2$$reg), ext::sxtw);
12011 %}
12012 ins_pipe(ialu_reg_reg);
12013 %}
12014
12015 // This pattern is automatically generated from aarch64_ad.m4
12016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12017 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12018 %{
12019 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12020 ins_cost(INSN_COST);
12021 format %{ "add $dst, $src1, $src2, sxtb" %}
12022
12023 ins_encode %{
12024 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12025 as_Register($src2$$reg), ext::sxtb);
12026 %}
12027 ins_pipe(ialu_reg_reg);
12028 %}
12029
12030 // This pattern is automatically generated from aarch64_ad.m4
12031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12032 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12033 %{
12034 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12035 ins_cost(INSN_COST);
12036 format %{ "add $dst, $src1, $src2, uxtb" %}
12037
12038 ins_encode %{
12039 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12040 as_Register($src2$$reg), ext::uxtb);
12041 %}
12042 ins_pipe(ialu_reg_reg);
12043 %}
12044
12045 // This pattern is automatically generated from aarch64_ad.m4
12046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12047 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12048 %{
12049 match(Set dst (AddI src1 (AndI src2 mask)));
12050 ins_cost(INSN_COST);
12051 format %{ "addw $dst, $src1, $src2, uxtb" %}
12052
12053 ins_encode %{
12054 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12055 as_Register($src2$$reg), ext::uxtb);
12056 %}
12057 ins_pipe(ialu_reg_reg);
12058 %}
12059
12060 // This pattern is automatically generated from aarch64_ad.m4
12061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12062 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12063 %{
12064 match(Set dst (AddI src1 (AndI src2 mask)));
12065 ins_cost(INSN_COST);
12066 format %{ "addw $dst, $src1, $src2, uxth" %}
12067
12068 ins_encode %{
12069 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12070 as_Register($src2$$reg), ext::uxth);
12071 %}
12072 ins_pipe(ialu_reg_reg);
12073 %}
12074
12075 // This pattern is automatically generated from aarch64_ad.m4
12076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12077 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12078 %{
12079 match(Set dst (AddL src1 (AndL src2 mask)));
12080 ins_cost(INSN_COST);
12081 format %{ "add $dst, $src1, $src2, uxtb" %}
12082
12083 ins_encode %{
12084 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12085 as_Register($src2$$reg), ext::uxtb);
12086 %}
12087 ins_pipe(ialu_reg_reg);
12088 %}
12089
12090 // This pattern is automatically generated from aarch64_ad.m4
12091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12092 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12093 %{
12094 match(Set dst (AddL src1 (AndL src2 mask)));
12095 ins_cost(INSN_COST);
12096 format %{ "add $dst, $src1, $src2, uxth" %}
12097
12098 ins_encode %{
12099 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12100 as_Register($src2$$reg), ext::uxth);
12101 %}
12102 ins_pipe(ialu_reg_reg);
12103 %}
12104
12105 // This pattern is automatically generated from aarch64_ad.m4
12106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12107 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12108 %{
12109 match(Set dst (AddL src1 (AndL src2 mask)));
12110 ins_cost(INSN_COST);
12111 format %{ "add $dst, $src1, $src2, uxtw" %}
12112
12113 ins_encode %{
12114 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12115 as_Register($src2$$reg), ext::uxtw);
12116 %}
12117 ins_pipe(ialu_reg_reg);
12118 %}
12119
12120 // This pattern is automatically generated from aarch64_ad.m4
12121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12122 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12123 %{
12124 match(Set dst (SubI src1 (AndI src2 mask)));
12125 ins_cost(INSN_COST);
12126 format %{ "subw $dst, $src1, $src2, uxtb" %}
12127
12128 ins_encode %{
12129 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12130 as_Register($src2$$reg), ext::uxtb);
12131 %}
12132 ins_pipe(ialu_reg_reg);
12133 %}
12134
12135 // This pattern is automatically generated from aarch64_ad.m4
12136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12137 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12138 %{
12139 match(Set dst (SubI src1 (AndI src2 mask)));
12140 ins_cost(INSN_COST);
12141 format %{ "subw $dst, $src1, $src2, uxth" %}
12142
12143 ins_encode %{
12144 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12145 as_Register($src2$$reg), ext::uxth);
12146 %}
12147 ins_pipe(ialu_reg_reg);
12148 %}
12149
12150 // This pattern is automatically generated from aarch64_ad.m4
12151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12152 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12153 %{
12154 match(Set dst (SubL src1 (AndL src2 mask)));
12155 ins_cost(INSN_COST);
12156 format %{ "sub $dst, $src1, $src2, uxtb" %}
12157
12158 ins_encode %{
12159 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12160 as_Register($src2$$reg), ext::uxtb);
12161 %}
12162 ins_pipe(ialu_reg_reg);
12163 %}
12164
12165 // This pattern is automatically generated from aarch64_ad.m4
12166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12167 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12168 %{
12169 match(Set dst (SubL src1 (AndL src2 mask)));
12170 ins_cost(INSN_COST);
12171 format %{ "sub $dst, $src1, $src2, uxth" %}
12172
12173 ins_encode %{
12174 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12175 as_Register($src2$$reg), ext::uxth);
12176 %}
12177 ins_pipe(ialu_reg_reg);
12178 %}
12179
12180 // This pattern is automatically generated from aarch64_ad.m4
12181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12182 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12183 %{
12184 match(Set dst (SubL src1 (AndL src2 mask)));
12185 ins_cost(INSN_COST);
12186 format %{ "sub $dst, $src1, $src2, uxtw" %}
12187
12188 ins_encode %{
12189 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12190 as_Register($src2$$reg), ext::uxtw);
12191 %}
12192 ins_pipe(ialu_reg_reg);
12193 %}
12194
12195
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12199 %{
12200 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12201 ins_cost(1.9 * INSN_COST);
12202 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12203
12204 ins_encode %{
12205 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12206 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12207 %}
12208 ins_pipe(ialu_reg_reg_shift);
12209 %}
12210
12211 // This pattern is automatically generated from aarch64_ad.m4
12212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12213 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12214 %{
12215 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12216 ins_cost(1.9 * INSN_COST);
12217 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12218
12219 ins_encode %{
12220 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12221 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12222 %}
12223 ins_pipe(ialu_reg_reg_shift);
12224 %}
12225
12226 // This pattern is automatically generated from aarch64_ad.m4
12227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12228 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12229 %{
12230 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12231 ins_cost(1.9 * INSN_COST);
12232 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12233
12234 ins_encode %{
12235 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12236 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12237 %}
12238 ins_pipe(ialu_reg_reg_shift);
12239 %}
12240
12241 // This pattern is automatically generated from aarch64_ad.m4
12242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12243 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12244 %{
12245 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12246 ins_cost(1.9 * INSN_COST);
12247 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12248
12249 ins_encode %{
12250 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12251 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12252 %}
12253 ins_pipe(ialu_reg_reg_shift);
12254 %}
12255
12256 // This pattern is automatically generated from aarch64_ad.m4
12257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12258 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12259 %{
12260 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12261 ins_cost(1.9 * INSN_COST);
12262 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12263
12264 ins_encode %{
12265 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12266 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12267 %}
12268 ins_pipe(ialu_reg_reg_shift);
12269 %}
12270
12271 // This pattern is automatically generated from aarch64_ad.m4
12272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12273 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12274 %{
12275 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12276 ins_cost(1.9 * INSN_COST);
12277 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12278
12279 ins_encode %{
12280 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12281 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12282 %}
12283 ins_pipe(ialu_reg_reg_shift);
12284 %}
12285
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12289 %{
12290 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12291 ins_cost(1.9 * INSN_COST);
12292 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12293
12294 ins_encode %{
12295 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12296 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12297 %}
12298 ins_pipe(ialu_reg_reg_shift);
12299 %}
12300
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12304 %{
12305 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12306 ins_cost(1.9 * INSN_COST);
12307 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12308
12309 ins_encode %{
12310 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12311 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12312 %}
12313 ins_pipe(ialu_reg_reg_shift);
12314 %}
12315
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12319 %{
12320 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12321 ins_cost(1.9 * INSN_COST);
12322 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12323
12324 ins_encode %{
12325 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12326 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12327 %}
12328 ins_pipe(ialu_reg_reg_shift);
12329 %}
12330
12331 // This pattern is automatically generated from aarch64_ad.m4
12332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12333 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12334 %{
12335 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12336 ins_cost(1.9 * INSN_COST);
12337 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12338
12339 ins_encode %{
12340 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12341 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12342 %}
12343 ins_pipe(ialu_reg_reg_shift);
12344 %}
12345
12346 // This pattern is automatically generated from aarch64_ad.m4
12347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12348 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12349 %{
12350 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12351 ins_cost(1.9 * INSN_COST);
12352 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12353
12354 ins_encode %{
12355 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12356 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12357 %}
12358 ins_pipe(ialu_reg_reg_shift);
12359 %}
12360
12361 // This pattern is automatically generated from aarch64_ad.m4
12362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12363 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12364 %{
12365 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12366 ins_cost(1.9 * INSN_COST);
12367 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12368
12369 ins_encode %{
12370 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12371 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12372 %}
12373 ins_pipe(ialu_reg_reg_shift);
12374 %}
12375
12376 // This pattern is automatically generated from aarch64_ad.m4
12377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12378 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12379 %{
12380 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12381 ins_cost(1.9 * INSN_COST);
12382 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12383
12384 ins_encode %{
12385 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12386 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12387 %}
12388 ins_pipe(ialu_reg_reg_shift);
12389 %}
12390
12391 // This pattern is automatically generated from aarch64_ad.m4
12392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12393 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12394 %{
12395 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12396 ins_cost(1.9 * INSN_COST);
12397 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12398
12399 ins_encode %{
12400 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12401 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12402 %}
12403 ins_pipe(ialu_reg_reg_shift);
12404 %}
12405
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12409 %{
12410 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12411 ins_cost(1.9 * INSN_COST);
12412 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12413
12414 ins_encode %{
12415 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12416 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12417 %}
12418 ins_pipe(ialu_reg_reg_shift);
12419 %}
12420
12421 // This pattern is automatically generated from aarch64_ad.m4
12422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12423 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12424 %{
12425 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12426 ins_cost(1.9 * INSN_COST);
12427 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12428
12429 ins_encode %{
12430 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12431 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12432 %}
12433 ins_pipe(ialu_reg_reg_shift);
12434 %}
12435
12436 // This pattern is automatically generated from aarch64_ad.m4
12437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12438 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12439 %{
12440 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12441 ins_cost(1.9 * INSN_COST);
12442 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12443
12444 ins_encode %{
12445 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12446 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12447 %}
12448 ins_pipe(ialu_reg_reg_shift);
12449 %}
12450
12451 // This pattern is automatically generated from aarch64_ad.m4
12452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12453 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12454 %{
12455 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12456 ins_cost(1.9 * INSN_COST);
12457 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12458
12459 ins_encode %{
12460 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12461 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12462 %}
12463 ins_pipe(ialu_reg_reg_shift);
12464 %}
12465
12466 // This pattern is automatically generated from aarch64_ad.m4
12467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12468 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12469 %{
12470 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12471 ins_cost(1.9 * INSN_COST);
12472 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12473
12474 ins_encode %{
12475 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12476 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12477 %}
12478 ins_pipe(ialu_reg_reg_shift);
12479 %}
12480
12481 // This pattern is automatically generated from aarch64_ad.m4
12482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12483 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12484 %{
12485 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12486 ins_cost(1.9 * INSN_COST);
12487 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12488
12489 ins_encode %{
12490 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12491 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12492 %}
12493 ins_pipe(ialu_reg_reg_shift);
12494 %}
12495
12496 // This pattern is automatically generated from aarch64_ad.m4
12497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12498 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12499 %{
12500 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12501 ins_cost(1.9 * INSN_COST);
12502 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12503
12504 ins_encode %{
12505 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12506 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12507 %}
12508 ins_pipe(ialu_reg_reg_shift);
12509 %}
12510
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12514 %{
12515 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12516 ins_cost(1.9 * INSN_COST);
12517 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12518
12519 ins_encode %{
12520 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12521 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12522 %}
12523 ins_pipe(ialu_reg_reg_shift);
12524 %}
12525
12526 // This pattern is automatically generated from aarch64_ad.m4
12527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12528 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12529 %{
12530 effect(DEF dst, USE src1, USE src2, USE cr);
12531 ins_cost(INSN_COST * 2);
12532 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12533
12534 ins_encode %{
12535 __ cselw($dst$$Register,
12536 $src1$$Register,
12537 $src2$$Register,
12538 Assembler::LT);
12539 %}
12540 ins_pipe(icond_reg_reg);
12541 %}
12542
12543 // This pattern is automatically generated from aarch64_ad.m4
12544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12545 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12546 %{
12547 effect(DEF dst, USE src1, USE src2, USE cr);
12548 ins_cost(INSN_COST * 2);
12549 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12550
12551 ins_encode %{
12552 __ cselw($dst$$Register,
12553 $src1$$Register,
12554 $src2$$Register,
12555 Assembler::GT);
12556 %}
12557 ins_pipe(icond_reg_reg);
12558 %}
12559
12560 // This pattern is automatically generated from aarch64_ad.m4
12561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12562 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12563 %{
12564 effect(DEF dst, USE src1, USE cr);
12565 ins_cost(INSN_COST * 2);
12566 format %{ "cselw $dst, $src1, zr lt\t" %}
12567
12568 ins_encode %{
12569 __ cselw($dst$$Register,
12570 $src1$$Register,
12571 zr,
12572 Assembler::LT);
12573 %}
12574 ins_pipe(icond_reg);
12575 %}
12576
12577 // This pattern is automatically generated from aarch64_ad.m4
12578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12579 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12580 %{
12581 effect(DEF dst, USE src1, USE cr);
12582 ins_cost(INSN_COST * 2);
12583 format %{ "cselw $dst, $src1, zr gt\t" %}
12584
12585 ins_encode %{
12586 __ cselw($dst$$Register,
12587 $src1$$Register,
12588 zr,
12589 Assembler::GT);
12590 %}
12591 ins_pipe(icond_reg);
12592 %}
12593
12594 // This pattern is automatically generated from aarch64_ad.m4
12595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12596 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12597 %{
12598 effect(DEF dst, USE src1, USE cr);
12599 ins_cost(INSN_COST * 2);
12600 format %{ "csincw $dst, $src1, zr le\t" %}
12601
12602 ins_encode %{
12603 __ csincw($dst$$Register,
12604 $src1$$Register,
12605 zr,
12606 Assembler::LE);
12607 %}
12608 ins_pipe(icond_reg);
12609 %}
12610
12611 // This pattern is automatically generated from aarch64_ad.m4
12612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12613 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12614 %{
12615 effect(DEF dst, USE src1, USE cr);
12616 ins_cost(INSN_COST * 2);
12617 format %{ "csincw $dst, $src1, zr gt\t" %}
12618
12619 ins_encode %{
12620 __ csincw($dst$$Register,
12621 $src1$$Register,
12622 zr,
12623 Assembler::GT);
12624 %}
12625 ins_pipe(icond_reg);
12626 %}
12627
12628 // This pattern is automatically generated from aarch64_ad.m4
12629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12630 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12631 %{
12632 effect(DEF dst, USE src1, USE cr);
12633 ins_cost(INSN_COST * 2);
12634 format %{ "csinvw $dst, $src1, zr lt\t" %}
12635
12636 ins_encode %{
12637 __ csinvw($dst$$Register,
12638 $src1$$Register,
12639 zr,
12640 Assembler::LT);
12641 %}
12642 ins_pipe(icond_reg);
12643 %}
12644
12645 // This pattern is automatically generated from aarch64_ad.m4
12646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12647 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12648 %{
12649 effect(DEF dst, USE src1, USE cr);
12650 ins_cost(INSN_COST * 2);
12651 format %{ "csinvw $dst, $src1, zr ge\t" %}
12652
12653 ins_encode %{
12654 __ csinvw($dst$$Register,
12655 $src1$$Register,
12656 zr,
12657 Assembler::GE);
12658 %}
12659 ins_pipe(icond_reg);
12660 %}
12661
12662 // This pattern is automatically generated from aarch64_ad.m4
12663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12664 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12665 %{
12666 match(Set dst (MinI src imm));
12667 ins_cost(INSN_COST * 3);
12668 expand %{
12669 rFlagsReg cr;
12670 compI_reg_imm0(cr, src);
12671 cmovI_reg_imm0_lt(dst, src, cr);
12672 %}
12673 %}
12674
12675 // This pattern is automatically generated from aarch64_ad.m4
12676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12677 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12678 %{
12679 match(Set dst (MinI imm src));
12680 ins_cost(INSN_COST * 3);
12681 expand %{
12682 rFlagsReg cr;
12683 compI_reg_imm0(cr, src);
12684 cmovI_reg_imm0_lt(dst, src, cr);
12685 %}
12686 %}
12687
12688 // This pattern is automatically generated from aarch64_ad.m4
12689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12690 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12691 %{
12692 match(Set dst (MinI src imm));
12693 ins_cost(INSN_COST * 3);
12694 expand %{
12695 rFlagsReg cr;
12696 compI_reg_imm0(cr, src);
12697 cmovI_reg_imm1_le(dst, src, cr);
12698 %}
12699 %}
12700
12701 // This pattern is automatically generated from aarch64_ad.m4
12702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12703 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12704 %{
12705 match(Set dst (MinI imm src));
12706 ins_cost(INSN_COST * 3);
12707 expand %{
12708 rFlagsReg cr;
12709 compI_reg_imm0(cr, src);
12710 cmovI_reg_imm1_le(dst, src, cr);
12711 %}
12712 %}
12713
12714 // This pattern is automatically generated from aarch64_ad.m4
12715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12716 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12717 %{
12718 match(Set dst (MinI src imm));
12719 ins_cost(INSN_COST * 3);
12720 expand %{
12721 rFlagsReg cr;
12722 compI_reg_imm0(cr, src);
12723 cmovI_reg_immM1_lt(dst, src, cr);
12724 %}
12725 %}
12726
12727 // This pattern is automatically generated from aarch64_ad.m4
12728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12729 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12730 %{
12731 match(Set dst (MinI imm src));
12732 ins_cost(INSN_COST * 3);
12733 expand %{
12734 rFlagsReg cr;
12735 compI_reg_imm0(cr, src);
12736 cmovI_reg_immM1_lt(dst, src, cr);
12737 %}
12738 %}
12739
12740 // This pattern is automatically generated from aarch64_ad.m4
12741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12742 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12743 %{
12744 match(Set dst (MaxI src imm));
12745 ins_cost(INSN_COST * 3);
12746 expand %{
12747 rFlagsReg cr;
12748 compI_reg_imm0(cr, src);
12749 cmovI_reg_imm0_gt(dst, src, cr);
12750 %}
12751 %}
12752
12753 // This pattern is automatically generated from aarch64_ad.m4
12754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12755 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12756 %{
12757 match(Set dst (MaxI imm src));
12758 ins_cost(INSN_COST * 3);
12759 expand %{
12760 rFlagsReg cr;
12761 compI_reg_imm0(cr, src);
12762 cmovI_reg_imm0_gt(dst, src, cr);
12763 %}
12764 %}
12765
12766 // This pattern is automatically generated from aarch64_ad.m4
12767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12768 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12769 %{
12770 match(Set dst (MaxI src imm));
12771 ins_cost(INSN_COST * 3);
12772 expand %{
12773 rFlagsReg cr;
12774 compI_reg_imm0(cr, src);
12775 cmovI_reg_imm1_gt(dst, src, cr);
12776 %}
12777 %}
12778
12779 // This pattern is automatically generated from aarch64_ad.m4
12780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12781 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12782 %{
12783 match(Set dst (MaxI imm src));
12784 ins_cost(INSN_COST * 3);
12785 expand %{
12786 rFlagsReg cr;
12787 compI_reg_imm0(cr, src);
12788 cmovI_reg_imm1_gt(dst, src, cr);
12789 %}
12790 %}
12791
12792 // This pattern is automatically generated from aarch64_ad.m4
12793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12794 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12795 %{
12796 match(Set dst (MaxI src imm));
12797 ins_cost(INSN_COST * 3);
12798 expand %{
12799 rFlagsReg cr;
12800 compI_reg_imm0(cr, src);
12801 cmovI_reg_immM1_ge(dst, src, cr);
12802 %}
12803 %}
12804
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12808 %{
12809 match(Set dst (MaxI imm src));
12810 ins_cost(INSN_COST * 3);
12811 expand %{
12812 rFlagsReg cr;
12813 compI_reg_imm0(cr, src);
12814 cmovI_reg_immM1_ge(dst, src, cr);
12815 %}
12816 %}
12817
12818 // This pattern is automatically generated from aarch64_ad.m4
12819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12820 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12821 %{
12822 match(Set dst (ReverseI src));
12823 ins_cost(INSN_COST);
12824 format %{ "rbitw $dst, $src" %}
12825 ins_encode %{
12826 __ rbitw($dst$$Register, $src$$Register);
12827 %}
12828 ins_pipe(ialu_reg);
12829 %}
12830
12831 // This pattern is automatically generated from aarch64_ad.m4
12832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12833 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12834 %{
12835 match(Set dst (ReverseL src));
12836 ins_cost(INSN_COST);
12837 format %{ "rbit $dst, $src" %}
12838 ins_encode %{
12839 __ rbit($dst$$Register, $src$$Register);
12840 %}
12841 ins_pipe(ialu_reg);
12842 %}
12843
12844
12845 // END This section of the file is automatically generated. Do not edit --------------
12846
12847
12848 // ============================================================================
12849 // Floating Point Arithmetic Instructions
12850
12851 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12852 match(Set dst (AddHF src1 src2));
12853 format %{ "faddh $dst, $src1, $src2" %}
12854 ins_encode %{
12855 __ faddh($dst$$FloatRegister,
12856 $src1$$FloatRegister,
12857 $src2$$FloatRegister);
12858 %}
12859 ins_pipe(fp_dop_reg_reg_s);
12860 %}
12861
12862 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12863 match(Set dst (AddF src1 src2));
12864
12865 ins_cost(INSN_COST * 5);
12866 format %{ "fadds $dst, $src1, $src2" %}
12867
12868 ins_encode %{
12869 __ fadds(as_FloatRegister($dst$$reg),
12870 as_FloatRegister($src1$$reg),
12871 as_FloatRegister($src2$$reg));
12872 %}
12873
12874 ins_pipe(fp_dop_reg_reg_s);
12875 %}
12876
12877 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12878 match(Set dst (AddD src1 src2));
12879
12880 ins_cost(INSN_COST * 5);
12881 format %{ "faddd $dst, $src1, $src2" %}
12882
12883 ins_encode %{
12884 __ faddd(as_FloatRegister($dst$$reg),
12885 as_FloatRegister($src1$$reg),
12886 as_FloatRegister($src2$$reg));
12887 %}
12888
12889 ins_pipe(fp_dop_reg_reg_d);
12890 %}
12891
12892 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12893 match(Set dst (SubHF src1 src2));
12894 format %{ "fsubh $dst, $src1, $src2" %}
12895 ins_encode %{
12896 __ fsubh($dst$$FloatRegister,
12897 $src1$$FloatRegister,
12898 $src2$$FloatRegister);
12899 %}
12900 ins_pipe(fp_dop_reg_reg_s);
12901 %}
12902
12903 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12904 match(Set dst (SubF src1 src2));
12905
12906 ins_cost(INSN_COST * 5);
12907 format %{ "fsubs $dst, $src1, $src2" %}
12908
12909 ins_encode %{
12910 __ fsubs(as_FloatRegister($dst$$reg),
12911 as_FloatRegister($src1$$reg),
12912 as_FloatRegister($src2$$reg));
12913 %}
12914
12915 ins_pipe(fp_dop_reg_reg_s);
12916 %}
12917
12918 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12919 match(Set dst (SubD src1 src2));
12920
12921 ins_cost(INSN_COST * 5);
12922 format %{ "fsubd $dst, $src1, $src2" %}
12923
12924 ins_encode %{
12925 __ fsubd(as_FloatRegister($dst$$reg),
12926 as_FloatRegister($src1$$reg),
12927 as_FloatRegister($src2$$reg));
12928 %}
12929
12930 ins_pipe(fp_dop_reg_reg_d);
12931 %}
12932
12933 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12934 match(Set dst (MulHF src1 src2));
12935 format %{ "fmulh $dst, $src1, $src2" %}
12936 ins_encode %{
12937 __ fmulh($dst$$FloatRegister,
12938 $src1$$FloatRegister,
12939 $src2$$FloatRegister);
12940 %}
12941 ins_pipe(fp_dop_reg_reg_s);
12942 %}
12943
12944 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12945 match(Set dst (MulF src1 src2));
12946
12947 ins_cost(INSN_COST * 6);
12948 format %{ "fmuls $dst, $src1, $src2" %}
12949
12950 ins_encode %{
12951 __ fmuls(as_FloatRegister($dst$$reg),
12952 as_FloatRegister($src1$$reg),
12953 as_FloatRegister($src2$$reg));
12954 %}
12955
12956 ins_pipe(fp_dop_reg_reg_s);
12957 %}
12958
12959 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12960 match(Set dst (MulD src1 src2));
12961
12962 ins_cost(INSN_COST * 6);
12963 format %{ "fmuld $dst, $src1, $src2" %}
12964
12965 ins_encode %{
12966 __ fmuld(as_FloatRegister($dst$$reg),
12967 as_FloatRegister($src1$$reg),
12968 as_FloatRegister($src2$$reg));
12969 %}
12970
12971 ins_pipe(fp_dop_reg_reg_d);
12972 %}
12973
12974 // src1 * src2 + src3 (half-precision float)
12975 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12976 match(Set dst (FmaHF src3 (Binary src1 src2)));
12977 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12978 ins_encode %{
12979 assert(UseFMA, "Needs FMA instructions support.");
12980 __ fmaddh($dst$$FloatRegister,
12981 $src1$$FloatRegister,
12982 $src2$$FloatRegister,
12983 $src3$$FloatRegister);
12984 %}
12985 ins_pipe(pipe_class_default);
12986 %}
12987
12988 // src1 * src2 + src3
12989 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12990 match(Set dst (FmaF src3 (Binary src1 src2)));
12991
12992 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12993
12994 ins_encode %{
12995 assert(UseFMA, "Needs FMA instructions support.");
12996 __ fmadds(as_FloatRegister($dst$$reg),
12997 as_FloatRegister($src1$$reg),
12998 as_FloatRegister($src2$$reg),
12999 as_FloatRegister($src3$$reg));
13000 %}
13001
13002 ins_pipe(pipe_class_default);
13003 %}
13004
13005 // src1 * src2 + src3
13006 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13007 match(Set dst (FmaD src3 (Binary src1 src2)));
13008
13009 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13010
13011 ins_encode %{
13012 assert(UseFMA, "Needs FMA instructions support.");
13013 __ fmaddd(as_FloatRegister($dst$$reg),
13014 as_FloatRegister($src1$$reg),
13015 as_FloatRegister($src2$$reg),
13016 as_FloatRegister($src3$$reg));
13017 %}
13018
13019 ins_pipe(pipe_class_default);
13020 %}
13021
13022 // src1 * (-src2) + src3
13023 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13024 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13025 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13026
13027 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13028
13029 ins_encode %{
13030 assert(UseFMA, "Needs FMA instructions support.");
13031 __ fmsubs(as_FloatRegister($dst$$reg),
13032 as_FloatRegister($src1$$reg),
13033 as_FloatRegister($src2$$reg),
13034 as_FloatRegister($src3$$reg));
13035 %}
13036
13037 ins_pipe(pipe_class_default);
13038 %}
13039
13040 // src1 * (-src2) + src3
13041 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13042 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13043 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13044
13045 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13046
13047 ins_encode %{
13048 assert(UseFMA, "Needs FMA instructions support.");
13049 __ fmsubd(as_FloatRegister($dst$$reg),
13050 as_FloatRegister($src1$$reg),
13051 as_FloatRegister($src2$$reg),
13052 as_FloatRegister($src3$$reg));
13053 %}
13054
13055 ins_pipe(pipe_class_default);
13056 %}
13057
13058 // src1 * (-src2) - src3
13059 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13060 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13061 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13062
13063 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13064
13065 ins_encode %{
13066 assert(UseFMA, "Needs FMA instructions support.");
13067 __ fnmadds(as_FloatRegister($dst$$reg),
13068 as_FloatRegister($src1$$reg),
13069 as_FloatRegister($src2$$reg),
13070 as_FloatRegister($src3$$reg));
13071 %}
13072
13073 ins_pipe(pipe_class_default);
13074 %}
13075
13076 // src1 * (-src2) - src3
13077 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13078 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13079 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13080
13081 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13082
13083 ins_encode %{
13084 assert(UseFMA, "Needs FMA instructions support.");
13085 __ fnmaddd(as_FloatRegister($dst$$reg),
13086 as_FloatRegister($src1$$reg),
13087 as_FloatRegister($src2$$reg),
13088 as_FloatRegister($src3$$reg));
13089 %}
13090
13091 ins_pipe(pipe_class_default);
13092 %}
13093
13094 // src1 * src2 - src3
13095 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13096 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13097
13098 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13099
13100 ins_encode %{
13101 assert(UseFMA, "Needs FMA instructions support.");
13102 __ fnmsubs(as_FloatRegister($dst$$reg),
13103 as_FloatRegister($src1$$reg),
13104 as_FloatRegister($src2$$reg),
13105 as_FloatRegister($src3$$reg));
13106 %}
13107
13108 ins_pipe(pipe_class_default);
13109 %}
13110
13111 // src1 * src2 - src3
13112 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13113 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13114
13115 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13116
13117 ins_encode %{
13118 assert(UseFMA, "Needs FMA instructions support.");
13119 // n.b. insn name should be fnmsubd
13120 __ fnmsub(as_FloatRegister($dst$$reg),
13121 as_FloatRegister($src1$$reg),
13122 as_FloatRegister($src2$$reg),
13123 as_FloatRegister($src3$$reg));
13124 %}
13125
13126 ins_pipe(pipe_class_default);
13127 %}
13128
13129 // Math.max(HH)H (half-precision float)
13130 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13131 match(Set dst (MaxHF src1 src2));
13132 format %{ "fmaxh $dst, $src1, $src2" %}
13133 ins_encode %{
13134 __ fmaxh($dst$$FloatRegister,
13135 $src1$$FloatRegister,
13136 $src2$$FloatRegister);
13137 %}
13138 ins_pipe(fp_dop_reg_reg_s);
13139 %}
13140
13141 // Math.min(HH)H (half-precision float)
13142 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13143 match(Set dst (MinHF src1 src2));
13144 format %{ "fminh $dst, $src1, $src2" %}
13145 ins_encode %{
13146 __ fminh($dst$$FloatRegister,
13147 $src1$$FloatRegister,
13148 $src2$$FloatRegister);
13149 %}
13150 ins_pipe(fp_dop_reg_reg_s);
13151 %}
13152
13153 // Math.max(FF)F
13154 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13155 match(Set dst (MaxF src1 src2));
13156
13157 format %{ "fmaxs $dst, $src1, $src2" %}
13158 ins_encode %{
13159 __ fmaxs(as_FloatRegister($dst$$reg),
13160 as_FloatRegister($src1$$reg),
13161 as_FloatRegister($src2$$reg));
13162 %}
13163
13164 ins_pipe(fp_dop_reg_reg_s);
13165 %}
13166
13167 // Math.min(FF)F
13168 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13169 match(Set dst (MinF src1 src2));
13170
13171 format %{ "fmins $dst, $src1, $src2" %}
13172 ins_encode %{
13173 __ fmins(as_FloatRegister($dst$$reg),
13174 as_FloatRegister($src1$$reg),
13175 as_FloatRegister($src2$$reg));
13176 %}
13177
13178 ins_pipe(fp_dop_reg_reg_s);
13179 %}
13180
13181 // Math.max(DD)D
13182 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13183 match(Set dst (MaxD src1 src2));
13184
13185 format %{ "fmaxd $dst, $src1, $src2" %}
13186 ins_encode %{
13187 __ fmaxd(as_FloatRegister($dst$$reg),
13188 as_FloatRegister($src1$$reg),
13189 as_FloatRegister($src2$$reg));
13190 %}
13191
13192 ins_pipe(fp_dop_reg_reg_d);
13193 %}
13194
13195 // Math.min(DD)D
13196 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13197 match(Set dst (MinD src1 src2));
13198
13199 format %{ "fmind $dst, $src1, $src2" %}
13200 ins_encode %{
13201 __ fmind(as_FloatRegister($dst$$reg),
13202 as_FloatRegister($src1$$reg),
13203 as_FloatRegister($src2$$reg));
13204 %}
13205
13206 ins_pipe(fp_dop_reg_reg_d);
13207 %}
13208
13209 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13210 match(Set dst (DivHF src1 src2));
13211 format %{ "fdivh $dst, $src1, $src2" %}
13212 ins_encode %{
13213 __ fdivh($dst$$FloatRegister,
13214 $src1$$FloatRegister,
13215 $src2$$FloatRegister);
13216 %}
13217 ins_pipe(fp_div_s);
13218 %}
13219
13220 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13221 match(Set dst (DivF src1 src2));
13222
13223 ins_cost(INSN_COST * 18);
13224 format %{ "fdivs $dst, $src1, $src2" %}
13225
13226 ins_encode %{
13227 __ fdivs(as_FloatRegister($dst$$reg),
13228 as_FloatRegister($src1$$reg),
13229 as_FloatRegister($src2$$reg));
13230 %}
13231
13232 ins_pipe(fp_div_s);
13233 %}
13234
13235 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13236 match(Set dst (DivD src1 src2));
13237
13238 ins_cost(INSN_COST * 32);
13239 format %{ "fdivd $dst, $src1, $src2" %}
13240
13241 ins_encode %{
13242 __ fdivd(as_FloatRegister($dst$$reg),
13243 as_FloatRegister($src1$$reg),
13244 as_FloatRegister($src2$$reg));
13245 %}
13246
13247 ins_pipe(fp_div_d);
13248 %}
13249
13250 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13251 match(Set dst (NegF src));
13252
13253 ins_cost(INSN_COST * 3);
13254 format %{ "fneg $dst, $src" %}
13255
13256 ins_encode %{
13257 __ fnegs(as_FloatRegister($dst$$reg),
13258 as_FloatRegister($src$$reg));
13259 %}
13260
13261 ins_pipe(fp_uop_s);
13262 %}
13263
13264 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13265 match(Set dst (NegD src));
13266
13267 ins_cost(INSN_COST * 3);
13268 format %{ "fnegd $dst, $src" %}
13269
13270 ins_encode %{
13271 __ fnegd(as_FloatRegister($dst$$reg),
13272 as_FloatRegister($src$$reg));
13273 %}
13274
13275 ins_pipe(fp_uop_d);
13276 %}
13277
13278 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13279 %{
13280 match(Set dst (AbsI src));
13281
13282 effect(KILL cr);
13283 ins_cost(INSN_COST * 2);
13284 format %{ "cmpw $src, zr\n\t"
13285 "cnegw $dst, $src, Assembler::LT\t# int abs"
13286 %}
13287
13288 ins_encode %{
13289 __ cmpw(as_Register($src$$reg), zr);
13290 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13291 %}
13292 ins_pipe(pipe_class_default);
13293 %}
13294
13295 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13296 %{
13297 match(Set dst (AbsL src));
13298
13299 effect(KILL cr);
13300 ins_cost(INSN_COST * 2);
13301 format %{ "cmp $src, zr\n\t"
13302 "cneg $dst, $src, Assembler::LT\t# long abs"
13303 %}
13304
13305 ins_encode %{
13306 __ cmp(as_Register($src$$reg), zr);
13307 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13308 %}
13309 ins_pipe(pipe_class_default);
13310 %}
13311
13312 instruct absF_reg(vRegF dst, vRegF src) %{
13313 match(Set dst (AbsF src));
13314
13315 ins_cost(INSN_COST * 3);
13316 format %{ "fabss $dst, $src" %}
13317 ins_encode %{
13318 __ fabss(as_FloatRegister($dst$$reg),
13319 as_FloatRegister($src$$reg));
13320 %}
13321
13322 ins_pipe(fp_uop_s);
13323 %}
13324
13325 instruct absD_reg(vRegD dst, vRegD src) %{
13326 match(Set dst (AbsD src));
13327
13328 ins_cost(INSN_COST * 3);
13329 format %{ "fabsd $dst, $src" %}
13330 ins_encode %{
13331 __ fabsd(as_FloatRegister($dst$$reg),
13332 as_FloatRegister($src$$reg));
13333 %}
13334
13335 ins_pipe(fp_uop_d);
13336 %}
13337
13338 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13339 match(Set dst (AbsF (SubF src1 src2)));
13340
13341 ins_cost(INSN_COST * 3);
13342 format %{ "fabds $dst, $src1, $src2" %}
13343 ins_encode %{
13344 __ fabds(as_FloatRegister($dst$$reg),
13345 as_FloatRegister($src1$$reg),
13346 as_FloatRegister($src2$$reg));
13347 %}
13348
13349 ins_pipe(fp_uop_s);
13350 %}
13351
13352 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13353 match(Set dst (AbsD (SubD src1 src2)));
13354
13355 ins_cost(INSN_COST * 3);
13356 format %{ "fabdd $dst, $src1, $src2" %}
13357 ins_encode %{
13358 __ fabdd(as_FloatRegister($dst$$reg),
13359 as_FloatRegister($src1$$reg),
13360 as_FloatRegister($src2$$reg));
13361 %}
13362
13363 ins_pipe(fp_uop_d);
13364 %}
13365
13366 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13367 match(Set dst (SqrtD src));
13368
13369 ins_cost(INSN_COST * 50);
13370 format %{ "fsqrtd $dst, $src" %}
13371 ins_encode %{
13372 __ fsqrtd(as_FloatRegister($dst$$reg),
13373 as_FloatRegister($src$$reg));
13374 %}
13375
13376 ins_pipe(fp_div_s);
13377 %}
13378
13379 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13380 match(Set dst (SqrtF src));
13381
13382 ins_cost(INSN_COST * 50);
13383 format %{ "fsqrts $dst, $src" %}
13384 ins_encode %{
13385 __ fsqrts(as_FloatRegister($dst$$reg),
13386 as_FloatRegister($src$$reg));
13387 %}
13388
13389 ins_pipe(fp_div_d);
13390 %}
13391
13392 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13393 match(Set dst (SqrtHF src));
13394 format %{ "fsqrth $dst, $src" %}
13395 ins_encode %{
13396 __ fsqrth($dst$$FloatRegister,
13397 $src$$FloatRegister);
13398 %}
13399 ins_pipe(fp_div_s);
13400 %}
13401
13402 // Math.rint, floor, ceil
13403 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13404 match(Set dst (RoundDoubleMode src rmode));
13405 format %{ "frint $dst, $src, $rmode" %}
13406 ins_encode %{
13407 switch ($rmode$$constant) {
13408 case RoundDoubleModeNode::rmode_rint:
13409 __ frintnd(as_FloatRegister($dst$$reg),
13410 as_FloatRegister($src$$reg));
13411 break;
13412 case RoundDoubleModeNode::rmode_floor:
13413 __ frintmd(as_FloatRegister($dst$$reg),
13414 as_FloatRegister($src$$reg));
13415 break;
13416 case RoundDoubleModeNode::rmode_ceil:
13417 __ frintpd(as_FloatRegister($dst$$reg),
13418 as_FloatRegister($src$$reg));
13419 break;
13420 }
13421 %}
13422 ins_pipe(fp_uop_d);
13423 %}
13424
13425 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13426 match(Set dst (CopySignD src1 (Binary src2 zero)));
13427 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13428 format %{ "CopySignD $dst $src1 $src2" %}
13429 ins_encode %{
13430 FloatRegister dst = as_FloatRegister($dst$$reg),
13431 src1 = as_FloatRegister($src1$$reg),
13432 src2 = as_FloatRegister($src2$$reg),
13433 zero = as_FloatRegister($zero$$reg);
13434 __ fnegd(dst, zero);
13435 __ bsl(dst, __ T8B, src2, src1);
13436 %}
13437 ins_pipe(fp_uop_d);
13438 %}
13439
13440 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13441 match(Set dst (CopySignF src1 src2));
13442 effect(TEMP_DEF dst, USE src1, USE src2);
13443 format %{ "CopySignF $dst $src1 $src2" %}
13444 ins_encode %{
13445 FloatRegister dst = as_FloatRegister($dst$$reg),
13446 src1 = as_FloatRegister($src1$$reg),
13447 src2 = as_FloatRegister($src2$$reg);
13448 __ movi(dst, __ T2S, 0x80, 24);
13449 __ bsl(dst, __ T8B, src2, src1);
13450 %}
13451 ins_pipe(fp_uop_d);
13452 %}
13453
13454 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13455 match(Set dst (SignumD src (Binary zero one)));
13456 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13457 format %{ "signumD $dst, $src" %}
13458 ins_encode %{
13459 FloatRegister src = as_FloatRegister($src$$reg),
13460 dst = as_FloatRegister($dst$$reg),
13461 zero = as_FloatRegister($zero$$reg),
13462 one = as_FloatRegister($one$$reg);
13463 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13464 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13465 // Bit selection instruction gets bit from "one" for each enabled bit in
13466 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13467 // NaN the whole "src" will be copied because "dst" is zero. For all other
13468 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13469 // from "src", and all other bits are copied from 1.0.
13470 __ bsl(dst, __ T8B, one, src);
13471 %}
13472 ins_pipe(fp_uop_d);
13473 %}
13474
13475 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13476 match(Set dst (SignumF src (Binary zero one)));
13477 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13478 format %{ "signumF $dst, $src" %}
13479 ins_encode %{
13480 FloatRegister src = as_FloatRegister($src$$reg),
13481 dst = as_FloatRegister($dst$$reg),
13482 zero = as_FloatRegister($zero$$reg),
13483 one = as_FloatRegister($one$$reg);
13484 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13485 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13486 // Bit selection instruction gets bit from "one" for each enabled bit in
13487 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13488 // NaN the whole "src" will be copied because "dst" is zero. For all other
13489 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13490 // from "src", and all other bits are copied from 1.0.
13491 __ bsl(dst, __ T8B, one, src);
13492 %}
13493 ins_pipe(fp_uop_d);
13494 %}
13495
13496 instruct onspinwait() %{
13497 match(OnSpinWait);
13498 ins_cost(INSN_COST);
13499
13500 format %{ "onspinwait" %}
13501
13502 ins_encode %{
13503 __ spin_wait();
13504 %}
13505 ins_pipe(pipe_class_empty);
13506 %}
13507
13508 // ============================================================================
13509 // Logical Instructions
13510
13511 // Integer Logical Instructions
13512
13513 // And Instructions
13514
13515
13516 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13517 match(Set dst (AndI src1 src2));
13518
13519 format %{ "andw $dst, $src1, $src2\t# int" %}
13520
13521 ins_cost(INSN_COST);
13522 ins_encode %{
13523 __ andw(as_Register($dst$$reg),
13524 as_Register($src1$$reg),
13525 as_Register($src2$$reg));
13526 %}
13527
13528 ins_pipe(ialu_reg_reg);
13529 %}
13530
13531 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13532 match(Set dst (AndI src1 src2));
13533
13534 format %{ "andsw $dst, $src1, $src2\t# int" %}
13535
13536 ins_cost(INSN_COST);
13537 ins_encode %{
13538 __ andw(as_Register($dst$$reg),
13539 as_Register($src1$$reg),
13540 (uint64_t)($src2$$constant));
13541 %}
13542
13543 ins_pipe(ialu_reg_imm);
13544 %}
13545
13546 // Or Instructions
13547
13548 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13549 match(Set dst (OrI src1 src2));
13550
13551 format %{ "orrw $dst, $src1, $src2\t# int" %}
13552
13553 ins_cost(INSN_COST);
13554 ins_encode %{
13555 __ orrw(as_Register($dst$$reg),
13556 as_Register($src1$$reg),
13557 as_Register($src2$$reg));
13558 %}
13559
13560 ins_pipe(ialu_reg_reg);
13561 %}
13562
13563 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13564 match(Set dst (OrI src1 src2));
13565
13566 format %{ "orrw $dst, $src1, $src2\t# int" %}
13567
13568 ins_cost(INSN_COST);
13569 ins_encode %{
13570 __ orrw(as_Register($dst$$reg),
13571 as_Register($src1$$reg),
13572 (uint64_t)($src2$$constant));
13573 %}
13574
13575 ins_pipe(ialu_reg_imm);
13576 %}
13577
13578 // Xor Instructions
13579
13580 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13581 match(Set dst (XorI src1 src2));
13582
13583 format %{ "eorw $dst, $src1, $src2\t# int" %}
13584
13585 ins_cost(INSN_COST);
13586 ins_encode %{
13587 __ eorw(as_Register($dst$$reg),
13588 as_Register($src1$$reg),
13589 as_Register($src2$$reg));
13590 %}
13591
13592 ins_pipe(ialu_reg_reg);
13593 %}
13594
13595 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13596 match(Set dst (XorI src1 src2));
13597
13598 format %{ "eorw $dst, $src1, $src2\t# int" %}
13599
13600 ins_cost(INSN_COST);
13601 ins_encode %{
13602 __ eorw(as_Register($dst$$reg),
13603 as_Register($src1$$reg),
13604 (uint64_t)($src2$$constant));
13605 %}
13606
13607 ins_pipe(ialu_reg_imm);
13608 %}
13609
13610 // Long Logical Instructions
13611 // TODO
13612
13613 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13614 match(Set dst (AndL src1 src2));
13615
13616 format %{ "and $dst, $src1, $src2\t# int" %}
13617
13618 ins_cost(INSN_COST);
13619 ins_encode %{
13620 __ andr(as_Register($dst$$reg),
13621 as_Register($src1$$reg),
13622 as_Register($src2$$reg));
13623 %}
13624
13625 ins_pipe(ialu_reg_reg);
13626 %}
13627
13628 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13629 match(Set dst (AndL src1 src2));
13630
13631 format %{ "and $dst, $src1, $src2\t# int" %}
13632
13633 ins_cost(INSN_COST);
13634 ins_encode %{
13635 __ andr(as_Register($dst$$reg),
13636 as_Register($src1$$reg),
13637 (uint64_t)($src2$$constant));
13638 %}
13639
13640 ins_pipe(ialu_reg_imm);
13641 %}
13642
13643 // Or Instructions
13644
13645 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13646 match(Set dst (OrL src1 src2));
13647
13648 format %{ "orr $dst, $src1, $src2\t# int" %}
13649
13650 ins_cost(INSN_COST);
13651 ins_encode %{
13652 __ orr(as_Register($dst$$reg),
13653 as_Register($src1$$reg),
13654 as_Register($src2$$reg));
13655 %}
13656
13657 ins_pipe(ialu_reg_reg);
13658 %}
13659
13660 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13661 match(Set dst (OrL src1 src2));
13662
13663 format %{ "orr $dst, $src1, $src2\t# int" %}
13664
13665 ins_cost(INSN_COST);
13666 ins_encode %{
13667 __ orr(as_Register($dst$$reg),
13668 as_Register($src1$$reg),
13669 (uint64_t)($src2$$constant));
13670 %}
13671
13672 ins_pipe(ialu_reg_imm);
13673 %}
13674
13675 // Xor Instructions
13676
13677 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13678 match(Set dst (XorL src1 src2));
13679
13680 format %{ "eor $dst, $src1, $src2\t# int" %}
13681
13682 ins_cost(INSN_COST);
13683 ins_encode %{
13684 __ eor(as_Register($dst$$reg),
13685 as_Register($src1$$reg),
13686 as_Register($src2$$reg));
13687 %}
13688
13689 ins_pipe(ialu_reg_reg);
13690 %}
13691
13692 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13693 match(Set dst (XorL src1 src2));
13694
13695 ins_cost(INSN_COST);
13696 format %{ "eor $dst, $src1, $src2\t# int" %}
13697
13698 ins_encode %{
13699 __ eor(as_Register($dst$$reg),
13700 as_Register($src1$$reg),
13701 (uint64_t)($src2$$constant));
13702 %}
13703
13704 ins_pipe(ialu_reg_imm);
13705 %}
13706
13707 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13708 %{
13709 match(Set dst (ConvI2L src));
13710
13711 ins_cost(INSN_COST);
13712 format %{ "sxtw $dst, $src\t# i2l" %}
13713 ins_encode %{
13714 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13715 %}
13716 ins_pipe(ialu_reg_shift);
13717 %}
13718
13719 // this pattern occurs in bigmath arithmetic
13720 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13721 %{
13722 match(Set dst (AndL (ConvI2L src) mask));
13723
13724 ins_cost(INSN_COST);
13725 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13726 ins_encode %{
13727 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13728 %}
13729
13730 ins_pipe(ialu_reg_shift);
13731 %}
13732
13733 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13734 match(Set dst (ConvL2I src));
13735
13736 ins_cost(INSN_COST);
13737 format %{ "movw $dst, $src \t// l2i" %}
13738
13739 ins_encode %{
13740 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13741 %}
13742
13743 ins_pipe(ialu_reg);
13744 %}
13745
13746 instruct convD2F_reg(vRegF dst, vRegD src) %{
13747 match(Set dst (ConvD2F src));
13748
13749 ins_cost(INSN_COST * 5);
13750 format %{ "fcvtd $dst, $src \t// d2f" %}
13751
13752 ins_encode %{
13753 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13754 %}
13755
13756 ins_pipe(fp_d2f);
13757 %}
13758
13759 instruct convF2D_reg(vRegD dst, vRegF src) %{
13760 match(Set dst (ConvF2D src));
13761
13762 ins_cost(INSN_COST * 5);
13763 format %{ "fcvts $dst, $src \t// f2d" %}
13764
13765 ins_encode %{
13766 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13767 %}
13768
13769 ins_pipe(fp_f2d);
13770 %}
13771
13772 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13773 match(Set dst (ConvF2I src));
13774
13775 ins_cost(INSN_COST * 5);
13776 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13777
13778 ins_encode %{
13779 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13780 %}
13781
13782 ins_pipe(fp_f2i);
13783 %}
13784
13785 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13786 match(Set dst (ConvF2L src));
13787
13788 ins_cost(INSN_COST * 5);
13789 format %{ "fcvtzs $dst, $src \t// f2l" %}
13790
13791 ins_encode %{
13792 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13793 %}
13794
13795 ins_pipe(fp_f2l);
13796 %}
13797
13798 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13799 match(Set dst (ConvF2HF src));
13800 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13801 "smov $dst, $tmp\t# move result from $tmp to $dst"
13802 %}
13803 effect(TEMP tmp);
13804 ins_encode %{
13805 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13806 %}
13807 ins_pipe(pipe_slow);
13808 %}
13809
13810 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13811 match(Set dst (ConvHF2F src));
13812 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13813 "fcvt $dst, $tmp\t# convert half to single precision"
13814 %}
13815 effect(TEMP tmp);
13816 ins_encode %{
13817 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13818 %}
13819 ins_pipe(pipe_slow);
13820 %}
13821
13822 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13823 match(Set dst (ConvI2F src));
13824
13825 ins_cost(INSN_COST * 5);
13826 format %{ "scvtfws $dst, $src \t// i2f" %}
13827
13828 ins_encode %{
13829 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13830 %}
13831
13832 ins_pipe(fp_i2f);
13833 %}
13834
13835 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13836 match(Set dst (ConvL2F src));
13837
13838 ins_cost(INSN_COST * 5);
13839 format %{ "scvtfs $dst, $src \t// l2f" %}
13840
13841 ins_encode %{
13842 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13843 %}
13844
13845 ins_pipe(fp_l2f);
13846 %}
13847
13848 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13849 match(Set dst (ConvD2I src));
13850
13851 ins_cost(INSN_COST * 5);
13852 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13853
13854 ins_encode %{
13855 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13856 %}
13857
13858 ins_pipe(fp_d2i);
13859 %}
13860
13861 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13862 match(Set dst (ConvD2L src));
13863
13864 ins_cost(INSN_COST * 5);
13865 format %{ "fcvtzd $dst, $src \t// d2l" %}
13866
13867 ins_encode %{
13868 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13869 %}
13870
13871 ins_pipe(fp_d2l);
13872 %}
13873
13874 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13875 match(Set dst (ConvI2D src));
13876
13877 ins_cost(INSN_COST * 5);
13878 format %{ "scvtfwd $dst, $src \t// i2d" %}
13879
13880 ins_encode %{
13881 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13882 %}
13883
13884 ins_pipe(fp_i2d);
13885 %}
13886
13887 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13888 match(Set dst (ConvL2D src));
13889
13890 ins_cost(INSN_COST * 5);
13891 format %{ "scvtfd $dst, $src \t// l2d" %}
13892
13893 ins_encode %{
13894 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13895 %}
13896
13897 ins_pipe(fp_l2d);
13898 %}
13899
13900 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13901 %{
13902 match(Set dst (RoundD src));
13903 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13904 format %{ "java_round_double $dst,$src"%}
13905 ins_encode %{
13906 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13907 as_FloatRegister($ftmp$$reg));
13908 %}
13909 ins_pipe(pipe_slow);
13910 %}
13911
13912 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13913 %{
13914 match(Set dst (RoundF src));
13915 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13916 format %{ "java_round_float $dst,$src"%}
13917 ins_encode %{
13918 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13919 as_FloatRegister($ftmp$$reg));
13920 %}
13921 ins_pipe(pipe_slow);
13922 %}
13923
13924 // stack <-> reg and reg <-> reg shuffles with no conversion
13925
13926 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13927
13928 match(Set dst (MoveF2I src));
13929
13930 effect(DEF dst, USE src);
13931
13932 ins_cost(4 * INSN_COST);
13933
13934 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13935
13936 ins_encode %{
13937 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13938 %}
13939
13940 ins_pipe(iload_reg_reg);
13941
13942 %}
13943
13944 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13945
13946 match(Set dst (MoveI2F src));
13947
13948 effect(DEF dst, USE src);
13949
13950 ins_cost(4 * INSN_COST);
13951
13952 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13953
13954 ins_encode %{
13955 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13956 %}
13957
13958 ins_pipe(pipe_class_memory);
13959
13960 %}
13961
13962 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13963
13964 match(Set dst (MoveD2L src));
13965
13966 effect(DEF dst, USE src);
13967
13968 ins_cost(4 * INSN_COST);
13969
13970 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13971
13972 ins_encode %{
13973 __ ldr($dst$$Register, Address(sp, $src$$disp));
13974 %}
13975
13976 ins_pipe(iload_reg_reg);
13977
13978 %}
13979
13980 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13981
13982 match(Set dst (MoveL2D src));
13983
13984 effect(DEF dst, USE src);
13985
13986 ins_cost(4 * INSN_COST);
13987
13988 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13989
13990 ins_encode %{
13991 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13992 %}
13993
13994 ins_pipe(pipe_class_memory);
13995
13996 %}
13997
13998 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13999
14000 match(Set dst (MoveF2I src));
14001
14002 effect(DEF dst, USE src);
14003
14004 ins_cost(INSN_COST);
14005
14006 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14007
14008 ins_encode %{
14009 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14010 %}
14011
14012 ins_pipe(pipe_class_memory);
14013
14014 %}
14015
14016 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14017
14018 match(Set dst (MoveI2F src));
14019
14020 effect(DEF dst, USE src);
14021
14022 ins_cost(INSN_COST);
14023
14024 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14025
14026 ins_encode %{
14027 __ strw($src$$Register, Address(sp, $dst$$disp));
14028 %}
14029
14030 ins_pipe(istore_reg_reg);
14031
14032 %}
14033
14034 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14035
14036 match(Set dst (MoveD2L src));
14037
14038 effect(DEF dst, USE src);
14039
14040 ins_cost(INSN_COST);
14041
14042 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14043
14044 ins_encode %{
14045 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14046 %}
14047
14048 ins_pipe(pipe_class_memory);
14049
14050 %}
14051
14052 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14053
14054 match(Set dst (MoveL2D src));
14055
14056 effect(DEF dst, USE src);
14057
14058 ins_cost(INSN_COST);
14059
14060 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14061
14062 ins_encode %{
14063 __ str($src$$Register, Address(sp, $dst$$disp));
14064 %}
14065
14066 ins_pipe(istore_reg_reg);
14067
14068 %}
14069
14070 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14071
14072 match(Set dst (MoveF2I src));
14073
14074 effect(DEF dst, USE src);
14075
14076 ins_cost(INSN_COST);
14077
14078 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14079
14080 ins_encode %{
14081 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14082 %}
14083
14084 ins_pipe(fp_f2i);
14085
14086 %}
14087
14088 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14089
14090 match(Set dst (MoveI2F src));
14091
14092 effect(DEF dst, USE src);
14093
14094 ins_cost(INSN_COST);
14095
14096 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14097
14098 ins_encode %{
14099 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14100 %}
14101
14102 ins_pipe(fp_i2f);
14103
14104 %}
14105
14106 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14107
14108 match(Set dst (MoveD2L src));
14109
14110 effect(DEF dst, USE src);
14111
14112 ins_cost(INSN_COST);
14113
14114 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14115
14116 ins_encode %{
14117 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14118 %}
14119
14120 ins_pipe(fp_d2l);
14121
14122 %}
14123
14124 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14125
14126 match(Set dst (MoveL2D src));
14127
14128 effect(DEF dst, USE src);
14129
14130 ins_cost(INSN_COST);
14131
14132 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14133
14134 ins_encode %{
14135 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14136 %}
14137
14138 ins_pipe(fp_l2d);
14139
14140 %}
14141
14142 // ============================================================================
14143 // clearing of an array
14144
14145 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14146 %{
14147 match(Set dummy (ClearArray (Binary cnt base) zero));
14148 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14149
14150 ins_cost(4 * INSN_COST);
14151 format %{ "ClearArray $cnt, $base" %}
14152
14153 ins_encode %{
14154 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14155 if (tpc == nullptr) {
14156 ciEnv::current()->record_failure("CodeCache is full");
14157 return;
14158 }
14159 %}
14160
14161 ins_pipe(pipe_class_memory);
14162 %}
14163
14164 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14165 %{
14166 predicate(((ClearArrayNode*)n)->word_copy_only());
14167 match(Set dummy (ClearArray (Binary cnt base) val));
14168 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14169
14170 ins_cost(4 * INSN_COST);
14171 format %{ "ClearArray $cnt, $base, $val" %}
14172
14173 ins_encode %{
14174 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14175 %}
14176
14177 ins_pipe(pipe_class_memory);
14178 %}
14179
14180 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14181 %{
14182 predicate((uint64_t)n->in(2)->get_long()
14183 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14184 && !((ClearArrayNode*)n)->word_copy_only());
14185 match(Set dummy (ClearArray cnt base));
14186 effect(TEMP temp, USE_KILL base, KILL cr);
14187
14188 ins_cost(4 * INSN_COST);
14189 format %{ "ClearArray $cnt, $base" %}
14190
14191 ins_encode %{
14192 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14193 if (tpc == nullptr) {
14194 ciEnv::current()->record_failure("CodeCache is full");
14195 return;
14196 }
14197 %}
14198
14199 ins_pipe(pipe_class_memory);
14200 %}
14201
14202 // ============================================================================
14203 // Overflow Math Instructions
14204
14205 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14206 %{
14207 match(Set cr (OverflowAddI op1 op2));
14208
14209 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14210 ins_cost(INSN_COST);
14211 ins_encode %{
14212 __ cmnw($op1$$Register, $op2$$Register);
14213 %}
14214
14215 ins_pipe(icmp_reg_reg);
14216 %}
14217
14218 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14219 %{
14220 match(Set cr (OverflowAddI op1 op2));
14221
14222 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14223 ins_cost(INSN_COST);
14224 ins_encode %{
14225 __ cmnw($op1$$Register, $op2$$constant);
14226 %}
14227
14228 ins_pipe(icmp_reg_imm);
14229 %}
14230
14231 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14232 %{
14233 match(Set cr (OverflowAddL op1 op2));
14234
14235 format %{ "cmn $op1, $op2\t# overflow check long" %}
14236 ins_cost(INSN_COST);
14237 ins_encode %{
14238 __ cmn($op1$$Register, $op2$$Register);
14239 %}
14240
14241 ins_pipe(icmp_reg_reg);
14242 %}
14243
14244 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14245 %{
14246 match(Set cr (OverflowAddL op1 op2));
14247
14248 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14249 ins_cost(INSN_COST);
14250 ins_encode %{
14251 __ adds(zr, $op1$$Register, $op2$$constant);
14252 %}
14253
14254 ins_pipe(icmp_reg_imm);
14255 %}
14256
14257 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14258 %{
14259 match(Set cr (OverflowSubI op1 op2));
14260
14261 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14262 ins_cost(INSN_COST);
14263 ins_encode %{
14264 __ cmpw($op1$$Register, $op2$$Register);
14265 %}
14266
14267 ins_pipe(icmp_reg_reg);
14268 %}
14269
14270 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14271 %{
14272 match(Set cr (OverflowSubI op1 op2));
14273
14274 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14275 ins_cost(INSN_COST);
14276 ins_encode %{
14277 __ cmpw($op1$$Register, $op2$$constant);
14278 %}
14279
14280 ins_pipe(icmp_reg_imm);
14281 %}
14282
14283 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14284 %{
14285 match(Set cr (OverflowSubL op1 op2));
14286
14287 format %{ "cmp $op1, $op2\t# overflow check long" %}
14288 ins_cost(INSN_COST);
14289 ins_encode %{
14290 __ cmp($op1$$Register, $op2$$Register);
14291 %}
14292
14293 ins_pipe(icmp_reg_reg);
14294 %}
14295
14296 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14297 %{
14298 match(Set cr (OverflowSubL op1 op2));
14299
14300 format %{ "cmp $op1, $op2\t# overflow check long" %}
14301 ins_cost(INSN_COST);
14302 ins_encode %{
14303 __ subs(zr, $op1$$Register, $op2$$constant);
14304 %}
14305
14306 ins_pipe(icmp_reg_imm);
14307 %}
14308
14309 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14310 %{
14311 match(Set cr (OverflowSubI zero op1));
14312
14313 format %{ "cmpw zr, $op1\t# overflow check int" %}
14314 ins_cost(INSN_COST);
14315 ins_encode %{
14316 __ cmpw(zr, $op1$$Register);
14317 %}
14318
14319 ins_pipe(icmp_reg_imm);
14320 %}
14321
14322 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14323 %{
14324 match(Set cr (OverflowSubL zero op1));
14325
14326 format %{ "cmp zr, $op1\t# overflow check long" %}
14327 ins_cost(INSN_COST);
14328 ins_encode %{
14329 __ cmp(zr, $op1$$Register);
14330 %}
14331
14332 ins_pipe(icmp_reg_imm);
14333 %}
14334
14335 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14336 %{
14337 match(Set cr (OverflowMulI op1 op2));
14338
14339 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14340 "cmp rscratch1, rscratch1, sxtw\n\t"
14341 "movw rscratch1, #0x80000000\n\t"
14342 "cselw rscratch1, rscratch1, zr, NE\n\t"
14343 "cmpw rscratch1, #1" %}
14344 ins_cost(5 * INSN_COST);
14345 ins_encode %{
14346 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14347 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14348 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14349 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14350 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14351 %}
14352
14353 ins_pipe(pipe_slow);
14354 %}
14355
14356 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14357 %{
14358 match(If cmp (OverflowMulI op1 op2));
14359 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14360 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14361 effect(USE labl, KILL cr);
14362
14363 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14364 "cmp rscratch1, rscratch1, sxtw\n\t"
14365 "b$cmp $labl" %}
14366 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14367 ins_encode %{
14368 Label* L = $labl$$label;
14369 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14370 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14371 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14372 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14373 %}
14374
14375 ins_pipe(pipe_serial);
14376 %}
14377
14378 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14379 %{
14380 match(Set cr (OverflowMulL op1 op2));
14381
14382 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14383 "smulh rscratch2, $op1, $op2\n\t"
14384 "cmp rscratch2, rscratch1, ASR #63\n\t"
14385 "movw rscratch1, #0x80000000\n\t"
14386 "cselw rscratch1, rscratch1, zr, NE\n\t"
14387 "cmpw rscratch1, #1" %}
14388 ins_cost(6 * INSN_COST);
14389 ins_encode %{
14390 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14391 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14392 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14393 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14394 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14395 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14396 %}
14397
14398 ins_pipe(pipe_slow);
14399 %}
14400
14401 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14402 %{
14403 match(If cmp (OverflowMulL op1 op2));
14404 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14405 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14406 effect(USE labl, KILL cr);
14407
14408 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14409 "smulh rscratch2, $op1, $op2\n\t"
14410 "cmp rscratch2, rscratch1, ASR #63\n\t"
14411 "b$cmp $labl" %}
14412 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14413 ins_encode %{
14414 Label* L = $labl$$label;
14415 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14416 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14417 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14418 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14419 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14420 %}
14421
14422 ins_pipe(pipe_serial);
14423 %}
14424
14425 // ============================================================================
14426 // Compare Instructions
14427
14428 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14429 %{
14430 match(Set cr (CmpI op1 op2));
14431
14432 effect(DEF cr, USE op1, USE op2);
14433
14434 ins_cost(INSN_COST);
14435 format %{ "cmpw $op1, $op2" %}
14436
14437 ins_encode(aarch64_enc_cmpw(op1, op2));
14438
14439 ins_pipe(icmp_reg_reg);
14440 %}
14441
14442 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14443 %{
14444 match(Set cr (CmpI op1 zero));
14445
14446 effect(DEF cr, USE op1);
14447
14448 ins_cost(INSN_COST);
14449 format %{ "cmpw $op1, 0" %}
14450
14451 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14452
14453 ins_pipe(icmp_reg_imm);
14454 %}
14455
14456 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14457 %{
14458 match(Set cr (CmpI op1 op2));
14459
14460 effect(DEF cr, USE op1);
14461
14462 ins_cost(INSN_COST);
14463 format %{ "cmpw $op1, $op2" %}
14464
14465 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14466
14467 ins_pipe(icmp_reg_imm);
14468 %}
14469
14470 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14471 %{
14472 match(Set cr (CmpI op1 op2));
14473
14474 effect(DEF cr, USE op1);
14475
14476 ins_cost(INSN_COST * 2);
14477 format %{ "cmpw $op1, $op2" %}
14478
14479 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14480
14481 ins_pipe(icmp_reg_imm);
14482 %}
14483
14484 // Unsigned compare Instructions; really, same as signed compare
14485 // except it should only be used to feed an If or a CMovI which takes a
14486 // cmpOpU.
14487
14488 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14489 %{
14490 match(Set cr (CmpU op1 op2));
14491
14492 effect(DEF cr, USE op1, USE op2);
14493
14494 ins_cost(INSN_COST);
14495 format %{ "cmpw $op1, $op2\t# unsigned" %}
14496
14497 ins_encode(aarch64_enc_cmpw(op1, op2));
14498
14499 ins_pipe(icmp_reg_reg);
14500 %}
14501
14502 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14503 %{
14504 match(Set cr (CmpU op1 zero));
14505
14506 effect(DEF cr, USE op1);
14507
14508 ins_cost(INSN_COST);
14509 format %{ "cmpw $op1, #0\t# unsigned" %}
14510
14511 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14512
14513 ins_pipe(icmp_reg_imm);
14514 %}
14515
14516 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14517 %{
14518 match(Set cr (CmpU op1 op2));
14519
14520 effect(DEF cr, USE op1);
14521
14522 ins_cost(INSN_COST);
14523 format %{ "cmpw $op1, $op2\t# unsigned" %}
14524
14525 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14526
14527 ins_pipe(icmp_reg_imm);
14528 %}
14529
14530 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14531 %{
14532 match(Set cr (CmpU op1 op2));
14533
14534 effect(DEF cr, USE op1);
14535
14536 ins_cost(INSN_COST * 2);
14537 format %{ "cmpw $op1, $op2\t# unsigned" %}
14538
14539 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14540
14541 ins_pipe(icmp_reg_imm);
14542 %}
14543
14544 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14545 %{
14546 match(Set cr (CmpL op1 op2));
14547
14548 effect(DEF cr, USE op1, USE op2);
14549
14550 ins_cost(INSN_COST);
14551 format %{ "cmp $op1, $op2" %}
14552
14553 ins_encode(aarch64_enc_cmp(op1, op2));
14554
14555 ins_pipe(icmp_reg_reg);
14556 %}
14557
14558 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14559 %{
14560 match(Set cr (CmpL op1 zero));
14561
14562 effect(DEF cr, USE op1);
14563
14564 ins_cost(INSN_COST);
14565 format %{ "tst $op1" %}
14566
14567 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14568
14569 ins_pipe(icmp_reg_imm);
14570 %}
14571
14572 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14573 %{
14574 match(Set cr (CmpL op1 op2));
14575
14576 effect(DEF cr, USE op1);
14577
14578 ins_cost(INSN_COST);
14579 format %{ "cmp $op1, $op2" %}
14580
14581 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14582
14583 ins_pipe(icmp_reg_imm);
14584 %}
14585
14586 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14587 %{
14588 match(Set cr (CmpL op1 op2));
14589
14590 effect(DEF cr, USE op1);
14591
14592 ins_cost(INSN_COST * 2);
14593 format %{ "cmp $op1, $op2" %}
14594
14595 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14596
14597 ins_pipe(icmp_reg_imm);
14598 %}
14599
14600 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14601 %{
14602 match(Set cr (CmpUL op1 op2));
14603
14604 effect(DEF cr, USE op1, USE op2);
14605
14606 ins_cost(INSN_COST);
14607 format %{ "cmp $op1, $op2" %}
14608
14609 ins_encode(aarch64_enc_cmp(op1, op2));
14610
14611 ins_pipe(icmp_reg_reg);
14612 %}
14613
14614 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14615 %{
14616 match(Set cr (CmpUL op1 zero));
14617
14618 effect(DEF cr, USE op1);
14619
14620 ins_cost(INSN_COST);
14621 format %{ "tst $op1" %}
14622
14623 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14624
14625 ins_pipe(icmp_reg_imm);
14626 %}
14627
14628 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14629 %{
14630 match(Set cr (CmpUL op1 op2));
14631
14632 effect(DEF cr, USE op1);
14633
14634 ins_cost(INSN_COST);
14635 format %{ "cmp $op1, $op2" %}
14636
14637 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14638
14639 ins_pipe(icmp_reg_imm);
14640 %}
14641
14642 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14643 %{
14644 match(Set cr (CmpUL op1 op2));
14645
14646 effect(DEF cr, USE op1);
14647
14648 ins_cost(INSN_COST * 2);
14649 format %{ "cmp $op1, $op2" %}
14650
14651 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14652
14653 ins_pipe(icmp_reg_imm);
14654 %}
14655
14656 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14657 %{
14658 match(Set cr (CmpP op1 op2));
14659
14660 effect(DEF cr, USE op1, USE op2);
14661
14662 ins_cost(INSN_COST);
14663 format %{ "cmp $op1, $op2\t // ptr" %}
14664
14665 ins_encode(aarch64_enc_cmpp(op1, op2));
14666
14667 ins_pipe(icmp_reg_reg);
14668 %}
14669
14670 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14671 %{
14672 match(Set cr (CmpN op1 op2));
14673
14674 effect(DEF cr, USE op1, USE op2);
14675
14676 ins_cost(INSN_COST);
14677 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14678
14679 ins_encode(aarch64_enc_cmpn(op1, op2));
14680
14681 ins_pipe(icmp_reg_reg);
14682 %}
14683
14684 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14685 %{
14686 match(Set cr (CmpP op1 zero));
14687
14688 effect(DEF cr, USE op1, USE zero);
14689
14690 ins_cost(INSN_COST);
14691 format %{ "cmp $op1, 0\t // ptr" %}
14692
14693 ins_encode(aarch64_enc_testp(op1));
14694
14695 ins_pipe(icmp_reg_imm);
14696 %}
14697
14698 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14699 %{
14700 match(Set cr (CmpN op1 zero));
14701
14702 effect(DEF cr, USE op1, USE zero);
14703
14704 ins_cost(INSN_COST);
14705 format %{ "cmp $op1, 0\t // compressed ptr" %}
14706
14707 ins_encode(aarch64_enc_testn(op1));
14708
14709 ins_pipe(icmp_reg_imm);
14710 %}
14711
14712 // FP comparisons
14713 //
14714 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14715 // using normal cmpOp. See declaration of rFlagsReg for details.
14716
14717 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14718 %{
14719 match(Set cr (CmpF src1 src2));
14720
14721 ins_cost(3 * INSN_COST);
14722 format %{ "fcmps $src1, $src2" %}
14723
14724 ins_encode %{
14725 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14726 %}
14727
14728 ins_pipe(pipe_class_compare);
14729 %}
14730
14731 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14732 %{
14733 match(Set cr (CmpF src1 src2));
14734
14735 ins_cost(3 * INSN_COST);
14736 format %{ "fcmps $src1, 0.0" %}
14737
14738 ins_encode %{
14739 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14740 %}
14741
14742 ins_pipe(pipe_class_compare);
14743 %}
14744 // FROM HERE
14745
14746 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14747 %{
14748 match(Set cr (CmpD src1 src2));
14749
14750 ins_cost(3 * INSN_COST);
14751 format %{ "fcmpd $src1, $src2" %}
14752
14753 ins_encode %{
14754 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14755 %}
14756
14757 ins_pipe(pipe_class_compare);
14758 %}
14759
14760 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14761 %{
14762 match(Set cr (CmpD src1 src2));
14763
14764 ins_cost(3 * INSN_COST);
14765 format %{ "fcmpd $src1, 0.0" %}
14766
14767 ins_encode %{
14768 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14769 %}
14770
14771 ins_pipe(pipe_class_compare);
14772 %}
14773
14774 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14775 %{
14776 match(Set dst (CmpF3 src1 src2));
14777 effect(KILL cr);
14778
14779 ins_cost(5 * INSN_COST);
14780 format %{ "fcmps $src1, $src2\n\t"
14781 "csinvw($dst, zr, zr, eq\n\t"
14782 "csnegw($dst, $dst, $dst, lt)"
14783 %}
14784
14785 ins_encode %{
14786 Label done;
14787 FloatRegister s1 = as_FloatRegister($src1$$reg);
14788 FloatRegister s2 = as_FloatRegister($src2$$reg);
14789 Register d = as_Register($dst$$reg);
14790 __ fcmps(s1, s2);
14791 // installs 0 if EQ else -1
14792 __ csinvw(d, zr, zr, Assembler::EQ);
14793 // keeps -1 if less or unordered else installs 1
14794 __ csnegw(d, d, d, Assembler::LT);
14795 __ bind(done);
14796 %}
14797
14798 ins_pipe(pipe_class_default);
14799
14800 %}
14801
14802 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14803 %{
14804 match(Set dst (CmpD3 src1 src2));
14805 effect(KILL cr);
14806
14807 ins_cost(5 * INSN_COST);
14808 format %{ "fcmpd $src1, $src2\n\t"
14809 "csinvw($dst, zr, zr, eq\n\t"
14810 "csnegw($dst, $dst, $dst, lt)"
14811 %}
14812
14813 ins_encode %{
14814 Label done;
14815 FloatRegister s1 = as_FloatRegister($src1$$reg);
14816 FloatRegister s2 = as_FloatRegister($src2$$reg);
14817 Register d = as_Register($dst$$reg);
14818 __ fcmpd(s1, s2);
14819 // installs 0 if EQ else -1
14820 __ csinvw(d, zr, zr, Assembler::EQ);
14821 // keeps -1 if less or unordered else installs 1
14822 __ csnegw(d, d, d, Assembler::LT);
14823 __ bind(done);
14824 %}
14825 ins_pipe(pipe_class_default);
14826
14827 %}
14828
14829 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14830 %{
14831 match(Set dst (CmpF3 src1 zero));
14832 effect(KILL cr);
14833
14834 ins_cost(5 * INSN_COST);
14835 format %{ "fcmps $src1, 0.0\n\t"
14836 "csinvw($dst, zr, zr, eq\n\t"
14837 "csnegw($dst, $dst, $dst, lt)"
14838 %}
14839
14840 ins_encode %{
14841 Label done;
14842 FloatRegister s1 = as_FloatRegister($src1$$reg);
14843 Register d = as_Register($dst$$reg);
14844 __ fcmps(s1, 0.0);
14845 // installs 0 if EQ else -1
14846 __ csinvw(d, zr, zr, Assembler::EQ);
14847 // keeps -1 if less or unordered else installs 1
14848 __ csnegw(d, d, d, Assembler::LT);
14849 __ bind(done);
14850 %}
14851
14852 ins_pipe(pipe_class_default);
14853
14854 %}
14855
14856 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14857 %{
14858 match(Set dst (CmpD3 src1 zero));
14859 effect(KILL cr);
14860
14861 ins_cost(5 * INSN_COST);
14862 format %{ "fcmpd $src1, 0.0\n\t"
14863 "csinvw($dst, zr, zr, eq\n\t"
14864 "csnegw($dst, $dst, $dst, lt)"
14865 %}
14866
14867 ins_encode %{
14868 Label done;
14869 FloatRegister s1 = as_FloatRegister($src1$$reg);
14870 Register d = as_Register($dst$$reg);
14871 __ fcmpd(s1, 0.0);
14872 // installs 0 if EQ else -1
14873 __ csinvw(d, zr, zr, Assembler::EQ);
14874 // keeps -1 if less or unordered else installs 1
14875 __ csnegw(d, d, d, Assembler::LT);
14876 __ bind(done);
14877 %}
14878 ins_pipe(pipe_class_default);
14879
14880 %}
14881
14882 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14883 %{
14884 match(Set dst (CmpLTMask p q));
14885 effect(KILL cr);
14886
14887 ins_cost(3 * INSN_COST);
14888
14889 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14890 "csetw $dst, lt\n\t"
14891 "subw $dst, zr, $dst"
14892 %}
14893
14894 ins_encode %{
14895 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14896 __ csetw(as_Register($dst$$reg), Assembler::LT);
14897 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14898 %}
14899
14900 ins_pipe(ialu_reg_reg);
14901 %}
14902
14903 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14904 %{
14905 match(Set dst (CmpLTMask src zero));
14906 effect(KILL cr);
14907
14908 ins_cost(INSN_COST);
14909
14910 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14911
14912 ins_encode %{
14913 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14914 %}
14915
14916 ins_pipe(ialu_reg_shift);
14917 %}
14918
14919 // ============================================================================
14920 // Max and Min
14921
14922 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14923
14924 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14925 %{
14926 effect(DEF cr, USE src);
14927 ins_cost(INSN_COST);
14928 format %{ "cmpw $src, 0" %}
14929
14930 ins_encode %{
14931 __ cmpw($src$$Register, 0);
14932 %}
14933 ins_pipe(icmp_reg_imm);
14934 %}
14935
14936 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14937 %{
14938 match(Set dst (MinI src1 src2));
14939 ins_cost(INSN_COST * 3);
14940
14941 expand %{
14942 rFlagsReg cr;
14943 compI_reg_reg(cr, src1, src2);
14944 cmovI_reg_reg_lt(dst, src1, src2, cr);
14945 %}
14946 %}
14947
14948 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14949 %{
14950 match(Set dst (MaxI src1 src2));
14951 ins_cost(INSN_COST * 3);
14952
14953 expand %{
14954 rFlagsReg cr;
14955 compI_reg_reg(cr, src1, src2);
14956 cmovI_reg_reg_gt(dst, src1, src2, cr);
14957 %}
14958 %}
14959
14960
14961 // ============================================================================
14962 // Branch Instructions
14963
14964 // Direct Branch.
14965 instruct branch(label lbl)
14966 %{
14967 match(Goto);
14968
14969 effect(USE lbl);
14970
14971 ins_cost(BRANCH_COST);
14972 format %{ "b $lbl" %}
14973
14974 ins_encode(aarch64_enc_b(lbl));
14975
14976 ins_pipe(pipe_branch);
14977 %}
14978
14979 // Conditional Near Branch
14980 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14981 %{
14982 // Same match rule as `branchConFar'.
14983 match(If cmp cr);
14984
14985 effect(USE lbl);
14986
14987 ins_cost(BRANCH_COST);
14988 // If set to 1 this indicates that the current instruction is a
14989 // short variant of a long branch. This avoids using this
14990 // instruction in first-pass matching. It will then only be used in
14991 // the `Shorten_branches' pass.
14992 // ins_short_branch(1);
14993 format %{ "b$cmp $lbl" %}
14994
14995 ins_encode(aarch64_enc_br_con(cmp, lbl));
14996
14997 ins_pipe(pipe_branch_cond);
14998 %}
14999
15000 // Conditional Near Branch Unsigned
15001 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15002 %{
15003 // Same match rule as `branchConFar'.
15004 match(If cmp cr);
15005
15006 effect(USE lbl);
15007
15008 ins_cost(BRANCH_COST);
15009 // If set to 1 this indicates that the current instruction is a
15010 // short variant of a long branch. This avoids using this
15011 // instruction in first-pass matching. It will then only be used in
15012 // the `Shorten_branches' pass.
15013 // ins_short_branch(1);
15014 format %{ "b$cmp $lbl\t# unsigned" %}
15015
15016 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15017
15018 ins_pipe(pipe_branch_cond);
15019 %}
15020
15021 // Make use of CBZ and CBNZ. These instructions, as well as being
15022 // shorter than (cmp; branch), have the additional benefit of not
15023 // killing the flags.
15024
15025 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15026 match(If cmp (CmpI op1 op2));
15027 effect(USE labl);
15028
15029 ins_cost(BRANCH_COST);
15030 format %{ "cbw$cmp $op1, $labl" %}
15031 ins_encode %{
15032 Label* L = $labl$$label;
15033 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15034 if (cond == Assembler::EQ)
15035 __ cbzw($op1$$Register, *L);
15036 else
15037 __ cbnzw($op1$$Register, *L);
15038 %}
15039 ins_pipe(pipe_cmp_branch);
15040 %}
15041
15042 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15043 match(If cmp (CmpL op1 op2));
15044 effect(USE labl);
15045
15046 ins_cost(BRANCH_COST);
15047 format %{ "cb$cmp $op1, $labl" %}
15048 ins_encode %{
15049 Label* L = $labl$$label;
15050 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15051 if (cond == Assembler::EQ)
15052 __ cbz($op1$$Register, *L);
15053 else
15054 __ cbnz($op1$$Register, *L);
15055 %}
15056 ins_pipe(pipe_cmp_branch);
15057 %}
15058
15059 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15060 match(If cmp (CmpP op1 op2));
15061 effect(USE labl);
15062
15063 ins_cost(BRANCH_COST);
15064 format %{ "cb$cmp $op1, $labl" %}
15065 ins_encode %{
15066 Label* L = $labl$$label;
15067 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15068 if (cond == Assembler::EQ)
15069 __ cbz($op1$$Register, *L);
15070 else
15071 __ cbnz($op1$$Register, *L);
15072 %}
15073 ins_pipe(pipe_cmp_branch);
15074 %}
15075
15076 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15077 match(If cmp (CmpN op1 op2));
15078 effect(USE labl);
15079
15080 ins_cost(BRANCH_COST);
15081 format %{ "cbw$cmp $op1, $labl" %}
15082 ins_encode %{
15083 Label* L = $labl$$label;
15084 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15085 if (cond == Assembler::EQ)
15086 __ cbzw($op1$$Register, *L);
15087 else
15088 __ cbnzw($op1$$Register, *L);
15089 %}
15090 ins_pipe(pipe_cmp_branch);
15091 %}
15092
15093 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15094 match(If cmp (CmpP (DecodeN oop) zero));
15095 effect(USE labl);
15096
15097 ins_cost(BRANCH_COST);
15098 format %{ "cb$cmp $oop, $labl" %}
15099 ins_encode %{
15100 Label* L = $labl$$label;
15101 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15102 if (cond == Assembler::EQ)
15103 __ cbzw($oop$$Register, *L);
15104 else
15105 __ cbnzw($oop$$Register, *L);
15106 %}
15107 ins_pipe(pipe_cmp_branch);
15108 %}
15109
15110 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15111 match(If cmp (CmpU op1 op2));
15112 effect(USE labl);
15113
15114 ins_cost(BRANCH_COST);
15115 format %{ "cbw$cmp $op1, $labl" %}
15116 ins_encode %{
15117 Label* L = $labl$$label;
15118 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15119 if (cond == Assembler::EQ || cond == Assembler::LS) {
15120 __ cbzw($op1$$Register, *L);
15121 } else {
15122 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15123 __ cbnzw($op1$$Register, *L);
15124 }
15125 %}
15126 ins_pipe(pipe_cmp_branch);
15127 %}
15128
15129 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15130 match(If cmp (CmpUL op1 op2));
15131 effect(USE labl);
15132
15133 ins_cost(BRANCH_COST);
15134 format %{ "cb$cmp $op1, $labl" %}
15135 ins_encode %{
15136 Label* L = $labl$$label;
15137 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15138 if (cond == Assembler::EQ || cond == Assembler::LS) {
15139 __ cbz($op1$$Register, *L);
15140 } else {
15141 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15142 __ cbnz($op1$$Register, *L);
15143 }
15144 %}
15145 ins_pipe(pipe_cmp_branch);
15146 %}
15147
15148 // Test bit and Branch
15149
15150 // Patterns for short (< 32KiB) variants
15151 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15152 match(If cmp (CmpL op1 op2));
15153 effect(USE labl);
15154
15155 ins_cost(BRANCH_COST);
15156 format %{ "cb$cmp $op1, $labl # long" %}
15157 ins_encode %{
15158 Label* L = $labl$$label;
15159 Assembler::Condition cond =
15160 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15161 __ tbr(cond, $op1$$Register, 63, *L);
15162 %}
15163 ins_pipe(pipe_cmp_branch);
15164 ins_short_branch(1);
15165 %}
15166
15167 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15168 match(If cmp (CmpI op1 op2));
15169 effect(USE labl);
15170
15171 ins_cost(BRANCH_COST);
15172 format %{ "cb$cmp $op1, $labl # int" %}
15173 ins_encode %{
15174 Label* L = $labl$$label;
15175 Assembler::Condition cond =
15176 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15177 __ tbr(cond, $op1$$Register, 31, *L);
15178 %}
15179 ins_pipe(pipe_cmp_branch);
15180 ins_short_branch(1);
15181 %}
15182
15183 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15184 match(If cmp (CmpL (AndL op1 op2) op3));
15185 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15186 effect(USE labl);
15187
15188 ins_cost(BRANCH_COST);
15189 format %{ "tb$cmp $op1, $op2, $labl" %}
15190 ins_encode %{
15191 Label* L = $labl$$label;
15192 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15193 int bit = exact_log2_long($op2$$constant);
15194 __ tbr(cond, $op1$$Register, bit, *L);
15195 %}
15196 ins_pipe(pipe_cmp_branch);
15197 ins_short_branch(1);
15198 %}
15199
15200 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15201 match(If cmp (CmpI (AndI op1 op2) op3));
15202 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15203 effect(USE labl);
15204
15205 ins_cost(BRANCH_COST);
15206 format %{ "tb$cmp $op1, $op2, $labl" %}
15207 ins_encode %{
15208 Label* L = $labl$$label;
15209 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15210 int bit = exact_log2((juint)$op2$$constant);
15211 __ tbr(cond, $op1$$Register, bit, *L);
15212 %}
15213 ins_pipe(pipe_cmp_branch);
15214 ins_short_branch(1);
15215 %}
15216
15217 // And far variants
15218 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15219 match(If cmp (CmpL op1 op2));
15220 effect(USE labl);
15221
15222 ins_cost(BRANCH_COST);
15223 format %{ "cb$cmp $op1, $labl # long" %}
15224 ins_encode %{
15225 Label* L = $labl$$label;
15226 Assembler::Condition cond =
15227 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15228 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15229 %}
15230 ins_pipe(pipe_cmp_branch);
15231 %}
15232
15233 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15234 match(If cmp (CmpI op1 op2));
15235 effect(USE labl);
15236
15237 ins_cost(BRANCH_COST);
15238 format %{ "cb$cmp $op1, $labl # int" %}
15239 ins_encode %{
15240 Label* L = $labl$$label;
15241 Assembler::Condition cond =
15242 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15243 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15244 %}
15245 ins_pipe(pipe_cmp_branch);
15246 %}
15247
15248 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15249 match(If cmp (CmpL (AndL op1 op2) op3));
15250 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15251 effect(USE labl);
15252
15253 ins_cost(BRANCH_COST);
15254 format %{ "tb$cmp $op1, $op2, $labl" %}
15255 ins_encode %{
15256 Label* L = $labl$$label;
15257 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15258 int bit = exact_log2_long($op2$$constant);
15259 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15260 %}
15261 ins_pipe(pipe_cmp_branch);
15262 %}
15263
15264 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15265 match(If cmp (CmpI (AndI op1 op2) op3));
15266 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15267 effect(USE labl);
15268
15269 ins_cost(BRANCH_COST);
15270 format %{ "tb$cmp $op1, $op2, $labl" %}
15271 ins_encode %{
15272 Label* L = $labl$$label;
15273 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15274 int bit = exact_log2((juint)$op2$$constant);
15275 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15276 %}
15277 ins_pipe(pipe_cmp_branch);
15278 %}
15279
15280 // Test bits
15281
15282 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15283 match(Set cr (CmpL (AndL op1 op2) op3));
15284 predicate(Assembler::operand_valid_for_logical_immediate
15285 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15286
15287 ins_cost(INSN_COST);
15288 format %{ "tst $op1, $op2 # long" %}
15289 ins_encode %{
15290 __ tst($op1$$Register, $op2$$constant);
15291 %}
15292 ins_pipe(ialu_reg_reg);
15293 %}
15294
15295 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15296 match(Set cr (CmpI (AndI op1 op2) op3));
15297 predicate(Assembler::operand_valid_for_logical_immediate
15298 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15299
15300 ins_cost(INSN_COST);
15301 format %{ "tst $op1, $op2 # int" %}
15302 ins_encode %{
15303 __ tstw($op1$$Register, $op2$$constant);
15304 %}
15305 ins_pipe(ialu_reg_reg);
15306 %}
15307
15308 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15309 match(Set cr (CmpL (AndL op1 op2) op3));
15310
15311 ins_cost(INSN_COST);
15312 format %{ "tst $op1, $op2 # long" %}
15313 ins_encode %{
15314 __ tst($op1$$Register, $op2$$Register);
15315 %}
15316 ins_pipe(ialu_reg_reg);
15317 %}
15318
15319 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15320 match(Set cr (CmpI (AndI op1 op2) op3));
15321
15322 ins_cost(INSN_COST);
15323 format %{ "tstw $op1, $op2 # int" %}
15324 ins_encode %{
15325 __ tstw($op1$$Register, $op2$$Register);
15326 %}
15327 ins_pipe(ialu_reg_reg);
15328 %}
15329
15330
15331 // Conditional Far Branch
15332 // Conditional Far Branch Unsigned
15333 // TODO: fixme
15334
15335 // counted loop end branch near
15336 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15337 %{
15338 match(CountedLoopEnd cmp cr);
15339
15340 effect(USE lbl);
15341
15342 ins_cost(BRANCH_COST);
15343 // short variant.
15344 // ins_short_branch(1);
15345 format %{ "b$cmp $lbl \t// counted loop end" %}
15346
15347 ins_encode(aarch64_enc_br_con(cmp, lbl));
15348
15349 ins_pipe(pipe_branch);
15350 %}
15351
15352 // counted loop end branch far
15353 // TODO: fixme
15354
15355 // ============================================================================
15356 // inlined locking and unlocking
15357
15358 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15359 %{
15360 match(Set cr (FastLock object box));
15361 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15362
15363 ins_cost(5 * INSN_COST);
15364 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15365
15366 ins_encode %{
15367 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15368 %}
15369
15370 ins_pipe(pipe_serial);
15371 %}
15372
15373 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15374 %{
15375 match(Set cr (FastUnlock object box));
15376 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15377
15378 ins_cost(5 * INSN_COST);
15379 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15380
15381 ins_encode %{
15382 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15383 %}
15384
15385 ins_pipe(pipe_serial);
15386 %}
15387
15388 // ============================================================================
15389 // Safepoint Instructions
15390
15391 // TODO
15392 // provide a near and far version of this code
15393
15394 instruct safePoint(rFlagsReg cr, iRegP poll)
15395 %{
15396 match(SafePoint poll);
15397 effect(KILL cr);
15398
15399 format %{
15400 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15401 %}
15402 ins_encode %{
15403 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15404 %}
15405 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15406 %}
15407
15408
15409 // ============================================================================
15410 // Procedure Call/Return Instructions
15411
15412 // Call Java Static Instruction
15413
15414 instruct CallStaticJavaDirect(method meth)
15415 %{
15416 match(CallStaticJava);
15417
15418 effect(USE meth);
15419
15420 ins_cost(CALL_COST);
15421
15422 format %{ "call,static $meth \t// ==> " %}
15423
15424 ins_encode(aarch64_enc_java_static_call(meth),
15425 aarch64_enc_call_epilog);
15426
15427 ins_pipe(pipe_class_call);
15428 %}
15429
15430 // TO HERE
15431
15432 // Call Java Dynamic Instruction
15433 instruct CallDynamicJavaDirect(method meth)
15434 %{
15435 match(CallDynamicJava);
15436
15437 effect(USE meth);
15438
15439 ins_cost(CALL_COST);
15440
15441 format %{ "CALL,dynamic $meth \t// ==> " %}
15442
15443 ins_encode(aarch64_enc_java_dynamic_call(meth),
15444 aarch64_enc_call_epilog);
15445
15446 ins_pipe(pipe_class_call);
15447 %}
15448
15449 // Call Runtime Instruction
15450
15451 instruct CallRuntimeDirect(method meth)
15452 %{
15453 match(CallRuntime);
15454
15455 effect(USE meth);
15456
15457 ins_cost(CALL_COST);
15458
15459 format %{ "CALL, runtime $meth" %}
15460
15461 ins_encode( aarch64_enc_java_to_runtime(meth) );
15462
15463 ins_pipe(pipe_class_call);
15464 %}
15465
15466 // Call Runtime Instruction
15467
15468 instruct CallLeafDirect(method meth)
15469 %{
15470 match(CallLeaf);
15471
15472 effect(USE meth);
15473
15474 ins_cost(CALL_COST);
15475
15476 format %{ "CALL, runtime leaf $meth" %}
15477
15478 ins_encode( aarch64_enc_java_to_runtime(meth) );
15479
15480 ins_pipe(pipe_class_call);
15481 %}
15482
15483 // Call Runtime Instruction without safepoint and with vector arguments
15484 instruct CallLeafDirectVector(method meth)
15485 %{
15486 match(CallLeafVector);
15487
15488 effect(USE meth);
15489
15490 ins_cost(CALL_COST);
15491
15492 format %{ "CALL, runtime leaf vector $meth" %}
15493
15494 ins_encode(aarch64_enc_java_to_runtime(meth));
15495
15496 ins_pipe(pipe_class_call);
15497 %}
15498
15499 // Call Runtime Instruction
15500
15501 // entry point is null, target holds the address to call
15502 instruct CallLeafNoFPIndirect(iRegP target)
15503 %{
15504 predicate(n->as_Call()->entry_point() == nullptr);
15505
15506 match(CallLeafNoFP target);
15507
15508 ins_cost(CALL_COST);
15509
15510 format %{ "CALL, runtime leaf nofp indirect $target" %}
15511
15512 ins_encode %{
15513 __ blr($target$$Register);
15514 %}
15515
15516 ins_pipe(pipe_class_call);
15517 %}
15518
15519 instruct CallLeafNoFPDirect(method meth)
15520 %{
15521 predicate(n->as_Call()->entry_point() != nullptr);
15522
15523 match(CallLeafNoFP);
15524
15525 effect(USE meth);
15526
15527 ins_cost(CALL_COST);
15528
15529 format %{ "CALL, runtime leaf nofp $meth" %}
15530
15531 ins_encode( aarch64_enc_java_to_runtime(meth) );
15532
15533 ins_pipe(pipe_class_call);
15534 %}
15535
15536 // Tail Call; Jump from runtime stub to Java code.
15537 // Also known as an 'interprocedural jump'.
15538 // Target of jump will eventually return to caller.
15539 // TailJump below removes the return address.
15540 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15541 // emitted just above the TailCall which has reset rfp to the caller state.
15542 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15543 %{
15544 match(TailCall jump_target method_ptr);
15545
15546 ins_cost(CALL_COST);
15547
15548 format %{ "br $jump_target\t# $method_ptr holds method" %}
15549
15550 ins_encode(aarch64_enc_tail_call(jump_target));
15551
15552 ins_pipe(pipe_class_call);
15553 %}
15554
15555 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15556 %{
15557 match(TailJump jump_target ex_oop);
15558
15559 ins_cost(CALL_COST);
15560
15561 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15562
15563 ins_encode(aarch64_enc_tail_jmp(jump_target));
15564
15565 ins_pipe(pipe_class_call);
15566 %}
15567
15568 // Forward exception.
15569 instruct ForwardExceptionjmp()
15570 %{
15571 match(ForwardException);
15572 ins_cost(CALL_COST);
15573
15574 format %{ "b forward_exception_stub" %}
15575 ins_encode %{
15576 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15577 %}
15578 ins_pipe(pipe_class_call);
15579 %}
15580
15581 // Create exception oop: created by stack-crawling runtime code.
15582 // Created exception is now available to this handler, and is setup
15583 // just prior to jumping to this handler. No code emitted.
15584 // TODO check
15585 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15586 instruct CreateException(iRegP_R0 ex_oop)
15587 %{
15588 match(Set ex_oop (CreateEx));
15589
15590 format %{ " -- \t// exception oop; no code emitted" %}
15591
15592 size(0);
15593
15594 ins_encode( /*empty*/ );
15595
15596 ins_pipe(pipe_class_empty);
15597 %}
15598
15599 // Rethrow exception: The exception oop will come in the first
15600 // argument position. Then JUMP (not call) to the rethrow stub code.
15601 instruct RethrowException() %{
15602 match(Rethrow);
15603 ins_cost(CALL_COST);
15604
15605 format %{ "b rethrow_stub" %}
15606
15607 ins_encode( aarch64_enc_rethrow() );
15608
15609 ins_pipe(pipe_class_call);
15610 %}
15611
15612
15613 // Return Instruction
15614 // epilog node loads ret address into lr as part of frame pop
15615 instruct Ret()
15616 %{
15617 match(Return);
15618
15619 format %{ "ret\t// return register" %}
15620
15621 ins_encode( aarch64_enc_ret() );
15622
15623 ins_pipe(pipe_branch);
15624 %}
15625
15626 // Die now.
15627 instruct ShouldNotReachHere() %{
15628 match(Halt);
15629
15630 ins_cost(CALL_COST);
15631 format %{ "ShouldNotReachHere" %}
15632
15633 ins_encode %{
15634 if (is_reachable()) {
15635 const char* str = __ code_string(_halt_reason);
15636 __ stop(str);
15637 }
15638 %}
15639
15640 ins_pipe(pipe_class_default);
15641 %}
15642
15643 // ============================================================================
15644 // Partial Subtype Check
15645 //
15646 // superklass array for an instance of the superklass. Set a hidden
15647 // internal cache on a hit (cache is checked with exposed code in
15648 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15649 // encoding ALSO sets flags.
15650
15651 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15652 %{
15653 match(Set result (PartialSubtypeCheck sub super));
15654 predicate(!UseSecondarySupersTable);
15655 effect(KILL cr, KILL temp);
15656
15657 ins_cost(20 * INSN_COST); // slightly larger than the next version
15658 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15659
15660 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15661
15662 opcode(0x1); // Force zero of result reg on hit
15663
15664 ins_pipe(pipe_class_memory);
15665 %}
15666
15667 // Two versions of partialSubtypeCheck, both used when we need to
15668 // search for a super class in the secondary supers array. The first
15669 // is used when we don't know _a priori_ the class being searched
15670 // for. The second, far more common, is used when we do know: this is
15671 // used for instanceof, checkcast, and any case where C2 can determine
15672 // it by constant propagation.
15673
15674 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15675 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15676 rFlagsReg cr)
15677 %{
15678 match(Set result (PartialSubtypeCheck sub super));
15679 predicate(UseSecondarySupersTable);
15680 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15681
15682 ins_cost(10 * INSN_COST); // slightly larger than the next version
15683 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15684
15685 ins_encode %{
15686 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15687 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15688 $vtemp$$FloatRegister,
15689 $result$$Register, /*L_success*/nullptr);
15690 %}
15691
15692 ins_pipe(pipe_class_memory);
15693 %}
15694
15695 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15696 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15697 rFlagsReg cr)
15698 %{
15699 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15700 predicate(UseSecondarySupersTable);
15701 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15702
15703 ins_cost(5 * INSN_COST); // smaller than the next version
15704 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15705
15706 ins_encode %{
15707 bool success = false;
15708 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15709 if (InlineSecondarySupersTest) {
15710 success =
15711 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15712 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15713 $vtemp$$FloatRegister,
15714 $result$$Register,
15715 super_klass_slot);
15716 } else {
15717 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15718 success = (call != nullptr);
15719 }
15720 if (!success) {
15721 ciEnv::current()->record_failure("CodeCache is full");
15722 return;
15723 }
15724 %}
15725
15726 ins_pipe(pipe_class_memory);
15727 %}
15728
15729 // Intrisics for String.compareTo()
15730
15731 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15732 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15733 %{
15734 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15735 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15736 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15737
15738 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15739 ins_encode %{
15740 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15741 __ string_compare($str1$$Register, $str2$$Register,
15742 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15743 $tmp1$$Register, $tmp2$$Register,
15744 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15745 %}
15746 ins_pipe(pipe_class_memory);
15747 %}
15748
15749 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15750 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15751 %{
15752 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15753 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15754 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15755
15756 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15757 ins_encode %{
15758 __ string_compare($str1$$Register, $str2$$Register,
15759 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15760 $tmp1$$Register, $tmp2$$Register,
15761 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15762 %}
15763 ins_pipe(pipe_class_memory);
15764 %}
15765
15766 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15767 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15768 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15769 %{
15770 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15771 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15772 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15773 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15774
15775 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15776 ins_encode %{
15777 __ string_compare($str1$$Register, $str2$$Register,
15778 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15779 $tmp1$$Register, $tmp2$$Register,
15780 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15781 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15782 %}
15783 ins_pipe(pipe_class_memory);
15784 %}
15785
15786 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15787 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15788 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15789 %{
15790 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15791 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15792 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15793 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15794
15795 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15796 ins_encode %{
15797 __ string_compare($str1$$Register, $str2$$Register,
15798 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15799 $tmp1$$Register, $tmp2$$Register,
15800 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15801 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15802 %}
15803 ins_pipe(pipe_class_memory);
15804 %}
15805
15806 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15807 // these string_compare variants as NEON register type for convenience so that the prototype of
15808 // string_compare can be shared with all variants.
15809
15810 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15811 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15812 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15813 pRegGov_P1 pgtmp2, rFlagsReg cr)
15814 %{
15815 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15816 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15817 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15818 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15819
15820 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15821 ins_encode %{
15822 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15823 __ string_compare($str1$$Register, $str2$$Register,
15824 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15825 $tmp1$$Register, $tmp2$$Register,
15826 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15827 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15828 StrIntrinsicNode::LL);
15829 %}
15830 ins_pipe(pipe_class_memory);
15831 %}
15832
15833 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15834 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15835 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15836 pRegGov_P1 pgtmp2, rFlagsReg cr)
15837 %{
15838 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15839 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15840 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15841 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15842
15843 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15844 ins_encode %{
15845 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15846 __ string_compare($str1$$Register, $str2$$Register,
15847 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15848 $tmp1$$Register, $tmp2$$Register,
15849 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15850 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15851 StrIntrinsicNode::LU);
15852 %}
15853 ins_pipe(pipe_class_memory);
15854 %}
15855
15856 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15857 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15858 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15859 pRegGov_P1 pgtmp2, rFlagsReg cr)
15860 %{
15861 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15862 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15863 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15864 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15865
15866 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15867 ins_encode %{
15868 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15869 __ string_compare($str1$$Register, $str2$$Register,
15870 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15871 $tmp1$$Register, $tmp2$$Register,
15872 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15873 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15874 StrIntrinsicNode::UL);
15875 %}
15876 ins_pipe(pipe_class_memory);
15877 %}
15878
15879 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15880 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15881 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15882 pRegGov_P1 pgtmp2, rFlagsReg cr)
15883 %{
15884 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15885 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15886 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15887 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15888
15889 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15890 ins_encode %{
15891 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15892 __ string_compare($str1$$Register, $str2$$Register,
15893 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15894 $tmp1$$Register, $tmp2$$Register,
15895 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15896 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15897 StrIntrinsicNode::UU);
15898 %}
15899 ins_pipe(pipe_class_memory);
15900 %}
15901
15902 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15903 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15904 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15905 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15906 %{
15907 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15908 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15909 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15910 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15911 TEMP vtmp0, TEMP vtmp1, KILL cr);
15912 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15913 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15914
15915 ins_encode %{
15916 __ string_indexof($str1$$Register, $str2$$Register,
15917 $cnt1$$Register, $cnt2$$Register,
15918 $tmp1$$Register, $tmp2$$Register,
15919 $tmp3$$Register, $tmp4$$Register,
15920 $tmp5$$Register, $tmp6$$Register,
15921 -1, $result$$Register, StrIntrinsicNode::UU);
15922 %}
15923 ins_pipe(pipe_class_memory);
15924 %}
15925
15926 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15927 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15928 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15929 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15930 %{
15931 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15932 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15933 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15934 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15935 TEMP vtmp0, TEMP vtmp1, KILL cr);
15936 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15937 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15938
15939 ins_encode %{
15940 __ string_indexof($str1$$Register, $str2$$Register,
15941 $cnt1$$Register, $cnt2$$Register,
15942 $tmp1$$Register, $tmp2$$Register,
15943 $tmp3$$Register, $tmp4$$Register,
15944 $tmp5$$Register, $tmp6$$Register,
15945 -1, $result$$Register, StrIntrinsicNode::LL);
15946 %}
15947 ins_pipe(pipe_class_memory);
15948 %}
15949
15950 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15951 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15952 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15953 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15954 %{
15955 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15956 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15957 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15958 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15959 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15960 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15961 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15962
15963 ins_encode %{
15964 __ string_indexof($str1$$Register, $str2$$Register,
15965 $cnt1$$Register, $cnt2$$Register,
15966 $tmp1$$Register, $tmp2$$Register,
15967 $tmp3$$Register, $tmp4$$Register,
15968 $tmp5$$Register, $tmp6$$Register,
15969 -1, $result$$Register, StrIntrinsicNode::UL);
15970 %}
15971 ins_pipe(pipe_class_memory);
15972 %}
15973
15974 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15975 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15976 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15977 %{
15978 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15979 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15980 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15981 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15982 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15983 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15984
15985 ins_encode %{
15986 int icnt2 = (int)$int_cnt2$$constant;
15987 __ string_indexof($str1$$Register, $str2$$Register,
15988 $cnt1$$Register, zr,
15989 $tmp1$$Register, $tmp2$$Register,
15990 $tmp3$$Register, $tmp4$$Register, zr, zr,
15991 icnt2, $result$$Register, StrIntrinsicNode::UU);
15992 %}
15993 ins_pipe(pipe_class_memory);
15994 %}
15995
15996 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15997 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15998 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15999 %{
16000 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16001 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16002 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16003 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16004 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16005 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16006
16007 ins_encode %{
16008 int icnt2 = (int)$int_cnt2$$constant;
16009 __ string_indexof($str1$$Register, $str2$$Register,
16010 $cnt1$$Register, zr,
16011 $tmp1$$Register, $tmp2$$Register,
16012 $tmp3$$Register, $tmp4$$Register, zr, zr,
16013 icnt2, $result$$Register, StrIntrinsicNode::LL);
16014 %}
16015 ins_pipe(pipe_class_memory);
16016 %}
16017
16018 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16019 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16020 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16021 %{
16022 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16023 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16024 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16025 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16026 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16027 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16028
16029 ins_encode %{
16030 int icnt2 = (int)$int_cnt2$$constant;
16031 __ string_indexof($str1$$Register, $str2$$Register,
16032 $cnt1$$Register, zr,
16033 $tmp1$$Register, $tmp2$$Register,
16034 $tmp3$$Register, $tmp4$$Register, zr, zr,
16035 icnt2, $result$$Register, StrIntrinsicNode::UL);
16036 %}
16037 ins_pipe(pipe_class_memory);
16038 %}
16039
16040 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16041 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16042 iRegINoSp tmp3, rFlagsReg cr)
16043 %{
16044 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16045 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16046 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16047 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16048
16049 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16050
16051 ins_encode %{
16052 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16053 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16054 $tmp3$$Register);
16055 %}
16056 ins_pipe(pipe_class_memory);
16057 %}
16058
16059 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16060 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16061 iRegINoSp tmp3, rFlagsReg cr)
16062 %{
16063 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16064 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16065 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16066 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16067
16068 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16069
16070 ins_encode %{
16071 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16072 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16073 $tmp3$$Register);
16074 %}
16075 ins_pipe(pipe_class_memory);
16076 %}
16077
16078 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16079 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16080 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16081 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16082 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16083 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16084 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16085 ins_encode %{
16086 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16087 $result$$Register, $ztmp1$$FloatRegister,
16088 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16089 $ptmp$$PRegister, true /* isL */);
16090 %}
16091 ins_pipe(pipe_class_memory);
16092 %}
16093
16094 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16095 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16096 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16097 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16098 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16099 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16100 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16101 ins_encode %{
16102 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16103 $result$$Register, $ztmp1$$FloatRegister,
16104 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16105 $ptmp$$PRegister, false /* isL */);
16106 %}
16107 ins_pipe(pipe_class_memory);
16108 %}
16109
16110 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16111 iRegI_R0 result, rFlagsReg cr)
16112 %{
16113 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16114 match(Set result (StrEquals (Binary str1 str2) cnt));
16115 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16116
16117 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16118 ins_encode %{
16119 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16120 __ string_equals($str1$$Register, $str2$$Register,
16121 $result$$Register, $cnt$$Register);
16122 %}
16123 ins_pipe(pipe_class_memory);
16124 %}
16125
16126 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16127 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16128 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16129 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16130 iRegP_R10 tmp, rFlagsReg cr)
16131 %{
16132 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16133 match(Set result (AryEq ary1 ary2));
16134 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16135 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16136 TEMP vtmp6, TEMP vtmp7, KILL cr);
16137
16138 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16139 ins_encode %{
16140 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16141 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16142 $result$$Register, $tmp$$Register, 1);
16143 if (tpc == nullptr) {
16144 ciEnv::current()->record_failure("CodeCache is full");
16145 return;
16146 }
16147 %}
16148 ins_pipe(pipe_class_memory);
16149 %}
16150
16151 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16152 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16153 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16154 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16155 iRegP_R10 tmp, rFlagsReg cr)
16156 %{
16157 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16158 match(Set result (AryEq ary1 ary2));
16159 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16160 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16161 TEMP vtmp6, TEMP vtmp7, KILL cr);
16162
16163 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16164 ins_encode %{
16165 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16166 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16167 $result$$Register, $tmp$$Register, 2);
16168 if (tpc == nullptr) {
16169 ciEnv::current()->record_failure("CodeCache is full");
16170 return;
16171 }
16172 %}
16173 ins_pipe(pipe_class_memory);
16174 %}
16175
16176 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16177 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16178 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16179 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16180 %{
16181 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16182 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16183 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16184
16185 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16186 ins_encode %{
16187 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16188 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16189 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16190 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16191 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16192 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16193 (BasicType)$basic_type$$constant);
16194 if (tpc == nullptr) {
16195 ciEnv::current()->record_failure("CodeCache is full");
16196 return;
16197 }
16198 %}
16199 ins_pipe(pipe_class_memory);
16200 %}
16201
16202 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16203 %{
16204 match(Set result (CountPositives ary1 len));
16205 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16206 format %{ "count positives byte[] $ary1,$len -> $result" %}
16207 ins_encode %{
16208 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16209 if (tpc == nullptr) {
16210 ciEnv::current()->record_failure("CodeCache is full");
16211 return;
16212 }
16213 %}
16214 ins_pipe( pipe_slow );
16215 %}
16216
16217 // fast char[] to byte[] compression
16218 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16219 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16220 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16221 iRegI_R0 result, rFlagsReg cr)
16222 %{
16223 match(Set result (StrCompressedCopy src (Binary dst len)));
16224 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16225 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16226
16227 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16228 ins_encode %{
16229 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16230 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16231 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16232 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16233 %}
16234 ins_pipe(pipe_slow);
16235 %}
16236
16237 // fast byte[] to char[] inflation
16238 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16239 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16240 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16241 %{
16242 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16243 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16244 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16245 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16246
16247 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16248 ins_encode %{
16249 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16250 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16251 $vtmp2$$FloatRegister, $tmp$$Register);
16252 if (tpc == nullptr) {
16253 ciEnv::current()->record_failure("CodeCache is full");
16254 return;
16255 }
16256 %}
16257 ins_pipe(pipe_class_memory);
16258 %}
16259
16260 // encode char[] to byte[] in ISO_8859_1
16261 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16262 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16263 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16264 iRegI_R0 result, rFlagsReg cr)
16265 %{
16266 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16267 match(Set result (EncodeISOArray src (Binary dst len)));
16268 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16269 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16270
16271 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16272 ins_encode %{
16273 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16274 $result$$Register, false,
16275 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16276 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16277 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16278 %}
16279 ins_pipe(pipe_class_memory);
16280 %}
16281
16282 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16283 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16284 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16285 iRegI_R0 result, rFlagsReg cr)
16286 %{
16287 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16288 match(Set result (EncodeISOArray src (Binary dst len)));
16289 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16290 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16291
16292 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16293 ins_encode %{
16294 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16295 $result$$Register, true,
16296 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16297 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16298 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16299 %}
16300 ins_pipe(pipe_class_memory);
16301 %}
16302
16303 //----------------------------- CompressBits/ExpandBits ------------------------
16304
16305 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16306 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16307 match(Set dst (CompressBits src mask));
16308 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16309 format %{ "mov $tsrc, $src\n\t"
16310 "mov $tmask, $mask\n\t"
16311 "bext $tdst, $tsrc, $tmask\n\t"
16312 "mov $dst, $tdst"
16313 %}
16314 ins_encode %{
16315 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16316 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16317 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16318 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16319 %}
16320 ins_pipe(pipe_slow);
16321 %}
16322
16323 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16324 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16325 match(Set dst (CompressBits (LoadI mem) mask));
16326 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16327 format %{ "ldrs $tsrc, $mem\n\t"
16328 "ldrs $tmask, $mask\n\t"
16329 "bext $tdst, $tsrc, $tmask\n\t"
16330 "mov $dst, $tdst"
16331 %}
16332 ins_encode %{
16333 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16334 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16335 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16336 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16337 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16338 %}
16339 ins_pipe(pipe_slow);
16340 %}
16341
16342 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16343 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16344 match(Set dst (CompressBits src mask));
16345 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16346 format %{ "mov $tsrc, $src\n\t"
16347 "mov $tmask, $mask\n\t"
16348 "bext $tdst, $tsrc, $tmask\n\t"
16349 "mov $dst, $tdst"
16350 %}
16351 ins_encode %{
16352 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16353 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16354 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16355 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16356 %}
16357 ins_pipe(pipe_slow);
16358 %}
16359
16360 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16361 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16362 match(Set dst (CompressBits (LoadL mem) mask));
16363 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16364 format %{ "ldrd $tsrc, $mem\n\t"
16365 "ldrd $tmask, $mask\n\t"
16366 "bext $tdst, $tsrc, $tmask\n\t"
16367 "mov $dst, $tdst"
16368 %}
16369 ins_encode %{
16370 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16371 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16372 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16373 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16374 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16375 %}
16376 ins_pipe(pipe_slow);
16377 %}
16378
16379 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16380 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16381 match(Set dst (ExpandBits src mask));
16382 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16383 format %{ "mov $tsrc, $src\n\t"
16384 "mov $tmask, $mask\n\t"
16385 "bdep $tdst, $tsrc, $tmask\n\t"
16386 "mov $dst, $tdst"
16387 %}
16388 ins_encode %{
16389 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16390 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16391 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16392 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16393 %}
16394 ins_pipe(pipe_slow);
16395 %}
16396
16397 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16398 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16399 match(Set dst (ExpandBits (LoadI mem) mask));
16400 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16401 format %{ "ldrs $tsrc, $mem\n\t"
16402 "ldrs $tmask, $mask\n\t"
16403 "bdep $tdst, $tsrc, $tmask\n\t"
16404 "mov $dst, $tdst"
16405 %}
16406 ins_encode %{
16407 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16408 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16409 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16410 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16411 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16412 %}
16413 ins_pipe(pipe_slow);
16414 %}
16415
16416 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16417 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16418 match(Set dst (ExpandBits src mask));
16419 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16420 format %{ "mov $tsrc, $src\n\t"
16421 "mov $tmask, $mask\n\t"
16422 "bdep $tdst, $tsrc, $tmask\n\t"
16423 "mov $dst, $tdst"
16424 %}
16425 ins_encode %{
16426 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16427 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16428 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16429 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16430 %}
16431 ins_pipe(pipe_slow);
16432 %}
16433
16434
16435 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16436 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16437 match(Set dst (ExpandBits (LoadL mem) mask));
16438 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16439 format %{ "ldrd $tsrc, $mem\n\t"
16440 "ldrd $tmask, $mask\n\t"
16441 "bdep $tdst, $tsrc, $tmask\n\t"
16442 "mov $dst, $tdst"
16443 %}
16444 ins_encode %{
16445 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16446 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16447 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16448 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16449 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16450 %}
16451 ins_pipe(pipe_slow);
16452 %}
16453
16454 //----------------------------- Reinterpret ----------------------------------
16455 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16456 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16457 match(Set dst (ReinterpretHF2S src));
16458 format %{ "reinterpretHF2S $dst, $src" %}
16459 ins_encode %{
16460 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16461 %}
16462 ins_pipe(pipe_slow);
16463 %}
16464
16465 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16466 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16467 match(Set dst (ReinterpretS2HF src));
16468 format %{ "reinterpretS2HF $dst, $src" %}
16469 ins_encode %{
16470 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16471 %}
16472 ins_pipe(pipe_slow);
16473 %}
16474
16475 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16476 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16477 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16478 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16479 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16480 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16481 // can be omitted in this pattern, resulting in -
16482 // fcvt $dst, $src // Convert float to half-precision float
16483 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16484 %{
16485 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16486 format %{ "convF2HFAndS2HF $dst, $src" %}
16487 ins_encode %{
16488 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16489 %}
16490 ins_pipe(pipe_slow);
16491 %}
16492
16493 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16494 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16495 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16496 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16497 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16498 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16499 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16500 // resulting in -
16501 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16502 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16503 %{
16504 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16505 format %{ "convHF2SAndHF2F $dst, $src" %}
16506 ins_encode %{
16507 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16508 %}
16509 ins_pipe(pipe_slow);
16510 %}
16511
16512 // ============================================================================
16513 // This name is KNOWN by the ADLC and cannot be changed.
16514 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16515 // for this guy.
16516 instruct tlsLoadP(thread_RegP dst)
16517 %{
16518 match(Set dst (ThreadLocal));
16519
16520 ins_cost(0);
16521
16522 format %{ " -- \t// $dst=Thread::current(), empty" %}
16523
16524 size(0);
16525
16526 ins_encode( /*empty*/ );
16527
16528 ins_pipe(pipe_class_empty);
16529 %}
16530
16531 //----------PEEPHOLE RULES-----------------------------------------------------
16532 // These must follow all instruction definitions as they use the names
16533 // defined in the instructions definitions.
16534 //
16535 // peepmatch ( root_instr_name [preceding_instruction]* );
16536 //
16537 // peepconstraint %{
16538 // (instruction_number.operand_name relational_op instruction_number.operand_name
16539 // [, ...] );
16540 // // instruction numbers are zero-based using left to right order in peepmatch
16541 //
16542 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16543 // // provide an instruction_number.operand_name for each operand that appears
16544 // // in the replacement instruction's match rule
16545 //
16546 // ---------VM FLAGS---------------------------------------------------------
16547 //
16548 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16549 //
16550 // Each peephole rule is given an identifying number starting with zero and
16551 // increasing by one in the order seen by the parser. An individual peephole
16552 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16553 // on the command-line.
16554 //
16555 // ---------CURRENT LIMITATIONS----------------------------------------------
16556 //
16557 // Only match adjacent instructions in same basic block
16558 // Only equality constraints
16559 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16560 // Only one replacement instruction
16561 //
16562 // ---------EXAMPLE----------------------------------------------------------
16563 //
16564 // // pertinent parts of existing instructions in architecture description
16565 // instruct movI(iRegINoSp dst, iRegI src)
16566 // %{
16567 // match(Set dst (CopyI src));
16568 // %}
16569 //
16570 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16571 // %{
16572 // match(Set dst (AddI dst src));
16573 // effect(KILL cr);
16574 // %}
16575 //
16576 // // Change (inc mov) to lea
16577 // peephole %{
16578 // // increment preceded by register-register move
16579 // peepmatch ( incI_iReg movI );
16580 // // require that the destination register of the increment
16581 // // match the destination register of the move
16582 // peepconstraint ( 0.dst == 1.dst );
16583 // // construct a replacement instruction that sets
16584 // // the destination to ( move's source register + one )
16585 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16586 // %}
16587 //
16588
16589 // Implementation no longer uses movX instructions since
16590 // machine-independent system no longer uses CopyX nodes.
16591 //
16592 // peephole
16593 // %{
16594 // peepmatch (incI_iReg movI);
16595 // peepconstraint (0.dst == 1.dst);
16596 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16597 // %}
16598
16599 // peephole
16600 // %{
16601 // peepmatch (decI_iReg movI);
16602 // peepconstraint (0.dst == 1.dst);
16603 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16604 // %}
16605
16606 // peephole
16607 // %{
16608 // peepmatch (addI_iReg_imm movI);
16609 // peepconstraint (0.dst == 1.dst);
16610 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16611 // %}
16612
16613 // peephole
16614 // %{
16615 // peepmatch (incL_iReg movL);
16616 // peepconstraint (0.dst == 1.dst);
16617 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16618 // %}
16619
16620 // peephole
16621 // %{
16622 // peepmatch (decL_iReg movL);
16623 // peepconstraint (0.dst == 1.dst);
16624 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16625 // %}
16626
16627 // peephole
16628 // %{
16629 // peepmatch (addL_iReg_imm movL);
16630 // peepconstraint (0.dst == 1.dst);
16631 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16632 // %}
16633
16634 // peephole
16635 // %{
16636 // peepmatch (addP_iReg_imm movP);
16637 // peepconstraint (0.dst == 1.dst);
16638 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16639 // %}
16640
16641 // // Change load of spilled value to only a spill
16642 // instruct storeI(memory mem, iRegI src)
16643 // %{
16644 // match(Set mem (StoreI mem src));
16645 // %}
16646 //
16647 // instruct loadI(iRegINoSp dst, memory mem)
16648 // %{
16649 // match(Set dst (LoadI mem));
16650 // %}
16651 //
16652
16653 //----------SMARTSPILL RULES---------------------------------------------------
16654 // These must follow all instruction definitions as they use the names
16655 // defined in the instructions definitions.
16656
16657 // Local Variables:
16658 // mode: c++
16659 // End: