1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2237 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2238 st->print_cr("\tcmpw rscratch1, r10");
2239 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2240 }
2241 #endif
2242
2243 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2244 {
2245 __ ic_check(InteriorEntryAlignment);
2246 }
2247
2248 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2249 {
2250 return MachNode::size(ra_);
2251 }
2252
2253 // REQUIRED EMIT CODE
2254
2255 //=============================================================================
2256
2257 // Emit deopt handler code.
2258 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2259 {
2260 // Note that the code buffer's insts_mark is always relative to insts.
2261 // That's why we must use the macroassembler to generate a handler.
2262 address base = __ start_a_stub(size_deopt_handler());
2263 if (base == nullptr) {
2264 ciEnv::current()->record_failure("CodeCache is full");
2265 return 0; // CodeBuffer::expand failed
2266 }
2267
2268 int offset = __ offset();
2269 Label start;
2270 __ bind(start);
2271 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2272
2273 int entry_offset = __ offset();
2274 __ b(start);
2275
2276 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2277 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2278 "out of bounds read in post-call NOP check");
2279 __ end_a_stub();
2280 return entry_offset;
2281 }
2282
2283 // REQUIRED MATCHER CODE
2284
2285 //=============================================================================
2286
2287 bool Matcher::match_rule_supported(int opcode) {
2288 if (!has_match_rule(opcode))
2289 return false;
2290
2291 switch (opcode) {
2292 case Op_OnSpinWait:
2293 return VM_Version::supports_on_spin_wait();
2294 case Op_CacheWB:
2295 case Op_CacheWBPreSync:
2296 case Op_CacheWBPostSync:
2297 if (!VM_Version::supports_data_cache_line_flush()) {
2298 return false;
2299 }
2300 break;
2301 case Op_ExpandBits:
2302 case Op_CompressBits:
2303 if (!VM_Version::supports_svebitperm()) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaF:
2308 case Op_FmaD:
2309 case Op_FmaVF:
2310 case Op_FmaVD:
2311 if (!UseFMA) {
2312 return false;
2313 }
2314 break;
2315 case Op_FmaHF:
2316 // UseFMA flag also needs to be checked along with FEAT_FP16
2317 if (!UseFMA || !is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 case Op_AddHF:
2322 case Op_SubHF:
2323 case Op_MulHF:
2324 case Op_DivHF:
2325 case Op_MinHF:
2326 case Op_MaxHF:
2327 case Op_SqrtHF:
2328 // Half-precision floating point scalar operations require FEAT_FP16
2329 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2330 // features are supported.
2331 if (!is_feat_fp16_supported()) {
2332 return false;
2333 }
2334 break;
2335 }
2336
2337 return true; // Per default match rules are supported.
2338 }
2339
2340 const RegMask* Matcher::predicate_reg_mask(void) {
2341 return &_PR_REG_mask;
2342 }
2343
2344 bool Matcher::supports_vector_calling_convention(void) {
2345 return EnableVectorSupport;
2346 }
2347
2348 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2349 assert(EnableVectorSupport, "sanity");
2350 int lo = V0_num;
2351 int hi = V0_H_num;
2352 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2353 hi = V0_K_num;
2354 }
2355 return OptoRegPair(hi, lo);
2356 }
2357
2358 // Is this branch offset short enough that a short branch can be used?
2359 //
2360 // NOTE: If the platform does not provide any short branch variants, then
2361 // this method should return false for offset 0.
2362 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2363 // The passed offset is relative to address of the branch.
2364
2365 return (-32768 <= offset && offset < 32768);
2366 }
2367
2368 // Vector width in bytes.
2369 int Matcher::vector_width_in_bytes(BasicType bt) {
2370 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2371 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2372 // Minimum 2 values in vector
2373 if (size < 2*type2aelembytes(bt)) size = 0;
2374 // But never < 4
2375 if (size < 4) size = 0;
2376 return size;
2377 }
2378
2379 // Limits on vector size (number of elements) loaded into vector.
2380 int Matcher::max_vector_size(const BasicType bt) {
2381 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2382 }
2383
2384 int Matcher::min_vector_size(const BasicType bt) {
2385 // Usually, the shortest vector length supported by AArch64 ISA and
2386 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2387 // vectors in a few special cases.
2388 int size;
2389 switch(bt) {
2390 case T_BOOLEAN:
2391 // Load/store a vector mask with only 2 elements for vector types
2392 // such as "2I/2F/2L/2D".
2393 size = 2;
2394 break;
2395 case T_BYTE:
2396 // Generate a "4B" vector, to support vector cast between "8B/16B"
2397 // and "4S/4I/4L/4F/4D".
2398 size = 4;
2399 break;
2400 case T_SHORT:
2401 // Generate a "2S" vector, to support vector cast between "4S/8S"
2402 // and "2I/2L/2F/2D".
2403 size = 2;
2404 break;
2405 default:
2406 // Limit the min vector length to 64-bit.
2407 size = 8 / type2aelembytes(bt);
2408 // The number of elements in a vector should be at least 2.
2409 size = MAX2(size, 2);
2410 }
2411
2412 int max_size = max_vector_size(bt);
2413 return MIN2(size, max_size);
2414 }
2415
2416 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2417 return Matcher::max_vector_size(bt);
2418 }
2419
2420 // Actual max scalable vector register length.
2421 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2422 return Matcher::max_vector_size(bt);
2423 }
2424
2425 // Vector ideal reg.
2426 uint Matcher::vector_ideal_reg(int len) {
2427 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2428 return Op_VecA;
2429 }
2430 switch(len) {
2431 // For 16-bit/32-bit mask vector, reuse VecD.
2432 case 2:
2433 case 4:
2434 case 8: return Op_VecD;
2435 case 16: return Op_VecX;
2436 }
2437 ShouldNotReachHere();
2438 return 0;
2439 }
2440
2441 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2442 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2443 switch (ideal_reg) {
2444 case Op_VecA: return new vecAOper();
2445 case Op_VecD: return new vecDOper();
2446 case Op_VecX: return new vecXOper();
2447 }
2448 ShouldNotReachHere();
2449 return nullptr;
2450 }
2451
2452 bool Matcher::is_reg2reg_move(MachNode* m) {
2453 return false;
2454 }
2455
2456 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2457 return false;
2458 }
2459
2460 bool Matcher::is_generic_vector(MachOper* opnd) {
2461 return opnd->opcode() == VREG;
2462 }
2463
2464 #ifdef ASSERT
2465 // Return whether or not this register is ever used as an argument.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486 #endif
2487
2488 uint Matcher::int_pressure_limit()
2489 {
2490 // JDK-8183543: When taking the number of available registers as int
2491 // register pressure threshold, the jtreg test:
2492 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2493 // failed due to C2 compilation failure with
2494 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2495 //
2496 // A derived pointer is live at CallNode and then is flagged by RA
2497 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2498 // derived pointers and lastly fail to spill after reaching maximum
2499 // number of iterations. Lowering the default pressure threshold to
2500 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2501 // a high register pressure area of the code so that split_DEF can
2502 // generate DefinitionSpillCopy for the derived pointer.
2503 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2504 if (!PreserveFramePointer) {
2505 // When PreserveFramePointer is off, frame pointer is allocatable,
2506 // but different from other SOC registers, it is excluded from
2507 // fatproj's mask because its save type is No-Save. Decrease 1 to
2508 // ensure high pressure at fatproj when PreserveFramePointer is off.
2509 // See check_pressure_at_fatproj().
2510 default_int_pressure_threshold--;
2511 }
2512 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2513 }
2514
2515 uint Matcher::float_pressure_limit()
2516 {
2517 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2518 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2519 }
2520
2521 const RegMask& Matcher::divI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for MODI projection of divmodI.
2527 const RegMask& Matcher::modI_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for DIVL projection of divmodL.
2533 const RegMask& Matcher::divL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 // Register for MODL projection of divmodL.
2539 const RegMask& Matcher::modL_proj_mask() {
2540 ShouldNotReachHere();
2541 return RegMask::EMPTY;
2542 }
2543
2544 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2545 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2546 Node* u = addp->fast_out(i);
2547 if (u->is_LoadStore()) {
2548 // On AArch64, LoadStoreNodes (i.e. compare and swap
2549 // instructions) only take register indirect as an operand, so
2550 // any attempt to use an AddPNode as an input to a LoadStoreNode
2551 // must fail.
2552 return false;
2553 }
2554 if (u->is_Mem()) {
2555 int opsize = u->as_Mem()->memory_size();
2556 assert(opsize > 0, "unexpected memory operand size");
2557 if (u->as_Mem()->memory_size() != (1<<shift)) {
2558 return false;
2559 }
2560 }
2561 }
2562 return true;
2563 }
2564
2565 // Convert BoolTest condition to Assembler condition.
2566 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2567 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2568 Assembler::Condition result;
2569 switch(cond) {
2570 case BoolTest::eq:
2571 result = Assembler::EQ; break;
2572 case BoolTest::ne:
2573 result = Assembler::NE; break;
2574 case BoolTest::le:
2575 result = Assembler::LE; break;
2576 case BoolTest::ge:
2577 result = Assembler::GE; break;
2578 case BoolTest::lt:
2579 result = Assembler::LT; break;
2580 case BoolTest::gt:
2581 result = Assembler::GT; break;
2582 case BoolTest::ule:
2583 result = Assembler::LS; break;
2584 case BoolTest::uge:
2585 result = Assembler::HS; break;
2586 case BoolTest::ult:
2587 result = Assembler::LO; break;
2588 case BoolTest::ugt:
2589 result = Assembler::HI; break;
2590 case BoolTest::overflow:
2591 result = Assembler::VS; break;
2592 case BoolTest::no_overflow:
2593 result = Assembler::VC; break;
2594 default:
2595 ShouldNotReachHere();
2596 return Assembler::Condition(-1);
2597 }
2598
2599 // Check conversion
2600 if (cond & BoolTest::unsigned_compare) {
2601 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2602 } else {
2603 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2604 }
2605
2606 return result;
2607 }
2608
2609 // Binary src (Replicate con)
2610 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2611 if (n == nullptr || m == nullptr) {
2612 return false;
2613 }
2614
2615 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2616 return false;
2617 }
2618
2619 Node* imm_node = m->in(1);
2620 if (!imm_node->is_Con()) {
2621 return false;
2622 }
2623
2624 const Type* t = imm_node->bottom_type();
2625 if (!(t->isa_int() || t->isa_long())) {
2626 return false;
2627 }
2628
2629 switch (n->Opcode()) {
2630 case Op_AndV:
2631 case Op_OrV:
2632 case Op_XorV: {
2633 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2634 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2635 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2636 }
2637 case Op_AddVB:
2638 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2639 case Op_AddVS:
2640 case Op_AddVI:
2641 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2642 case Op_AddVL:
2643 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2644 default:
2645 return false;
2646 }
2647 }
2648
2649 // (XorV src (Replicate m1))
2650 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2651 if (n != nullptr && m != nullptr) {
2652 return n->Opcode() == Op_XorV &&
2653 VectorNode::is_all_ones_vector(m);
2654 }
2655 return false;
2656 }
2657
2658 // Returns true if (n, m) matches "(XorVMask vm2 (MaskAll m1))" and that XorVMask
2659 // is used only by an AndVMask. In that case, cloning m (the MaskAll) lets the
2660 // matcher avoid sharing the MaskAll node and subsume the pattern into rule:
2661 // "(AndVMask vm1 (XorVMask vm2 (MaskAll m1)))".
2662 //
2663 // Limitation: the "andNot" rule still cannot be matched if "m" has other
2664 // uses outside this pattern.
2665 static bool is_vector_mask_not_operand_in_andnot_pattern(Node* n, Node* m) {
2666 if (n == nullptr || m == nullptr) {
2667 return false;
2668 }
2669
2670 if (VectorNode::is_all_ones_vector(m) &&
2671 n->Opcode() == Op_XorVMask &&
2672 n->outcnt() == 1 &&
2673 n->unique_out()->Opcode() == Op_AndVMask) {
2674 // If another input of the AndVMask is also a mask-not pattern that would
2675 // qualify for the `maskAll` cloning, do not clone the "maskAll" here,
2676 // because the match rule can only consume one such pattern.
2677 Node* use = n->unique_out();
2678 Node* other_input = use->in(1) == n ? use->in(2) : use->in(1);
2679 return !VectorNode::is_vectormask_bitwise_not_pattern(other_input);
2680 }
2681 return false;
2682 }
2683
2684 // Should the matcher clone input 'm' of node 'n'?
2685 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2686 if (is_vshift_con_pattern(n, m) ||
2687 is_vector_bitwise_not_pattern(n, m) ||
2688 is_vector_mask_not_operand_in_andnot_pattern(n, m) ||
2689 is_valid_sve_arith_imm_pattern(n, m) ||
2690 is_encode_and_store_pattern(n, m)) {
2691 mstack.push(m, Visit);
2692 return true;
2693 }
2694 return false;
2695 }
2696
2697 // Should the Matcher clone shifts on addressing modes, expecting them
2698 // to be subsumed into complex addressing expressions or compute them
2699 // into registers?
2700 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2701
2702 // Loads and stores with indirect memory input (e.g., volatile loads and
2703 // stores) do not subsume the input into complex addressing expressions. If
2704 // the addressing expression is input to at least one such load or store, do
2705 // not clone the addressing expression. Query needs_acquiring_load and
2706 // needs_releasing_store as a proxy for indirect memory input, as it is not
2707 // possible to directly query for indirect memory input at this stage.
2708 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2709 Node* n = m->fast_out(i);
2710 if (n->is_Load() && needs_acquiring_load(n)) {
2711 return false;
2712 }
2713 if (n->is_Store() && needs_releasing_store(n)) {
2714 return false;
2715 }
2716 }
2717
2718 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2719 return true;
2720 }
2721
2722 Node *off = m->in(AddPNode::Offset);
2723 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2724 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2725 // Are there other uses besides address expressions?
2726 !is_visited(off)) {
2727 address_visited.set(off->_idx); // Flag as address_visited
2728 mstack.push(off->in(2), Visit);
2729 Node *conv = off->in(1);
2730 if (conv->Opcode() == Op_ConvI2L &&
2731 // Are there other uses besides address expressions?
2732 !is_visited(conv)) {
2733 address_visited.set(conv->_idx); // Flag as address_visited
2734 mstack.push(conv->in(1), Pre_Visit);
2735 } else {
2736 mstack.push(conv, Pre_Visit);
2737 }
2738 address_visited.test_set(m->_idx); // Flag as address_visited
2739 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2740 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2741 return true;
2742 } else if (off->Opcode() == Op_ConvI2L &&
2743 // Are there other uses besides address expressions?
2744 !is_visited(off)) {
2745 address_visited.test_set(m->_idx); // Flag as address_visited
2746 address_visited.set(off->_idx); // Flag as address_visited
2747 mstack.push(off->in(1), Pre_Visit);
2748 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2749 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2750 return true;
2751 }
2752 return false;
2753 }
2754
2755 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2756 { \
2757 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2758 guarantee(DISP == 0, "mode not permitted for volatile"); \
2759 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2760 __ INSN(REG, as_Register(BASE)); \
2761 }
2762
2763
2764 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2765 {
2766 Address::extend scale;
2767
2768 // Hooboy, this is fugly. We need a way to communicate to the
2769 // encoder that the index needs to be sign extended, so we have to
2770 // enumerate all the cases.
2771 switch (opcode) {
2772 case INDINDEXSCALEDI2L:
2773 case INDINDEXSCALEDI2LN:
2774 case INDINDEXI2L:
2775 case INDINDEXI2LN:
2776 scale = Address::sxtw(size);
2777 break;
2778 default:
2779 scale = Address::lsl(size);
2780 }
2781
2782 if (index == -1) {
2783 return Address(base, disp);
2784 } else {
2785 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2786 return Address(base, as_Register(index), scale);
2787 }
2788 }
2789
2790
2791 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2792 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2793 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2794 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2795 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2796
2797 // Used for all non-volatile memory accesses. The use of
2798 // $mem->opcode() to discover whether this pattern uses sign-extended
2799 // offsets is something of a kludge.
2800 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2801 Register reg, int opcode,
2802 Register base, int index, int scale, int disp,
2803 int size_in_memory)
2804 {
2805 Address addr = mem2address(opcode, base, index, scale, disp);
2806 if (addr.getMode() == Address::base_plus_offset) {
2807 /* Fix up any out-of-range offsets. */
2808 assert_different_registers(rscratch1, base);
2809 assert_different_registers(rscratch1, reg);
2810 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2811 }
2812 (masm->*insn)(reg, addr);
2813 }
2814
2815 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2816 FloatRegister reg, int opcode,
2817 Register base, int index, int size, int disp,
2818 int size_in_memory)
2819 {
2820 Address::extend scale;
2821
2822 switch (opcode) {
2823 case INDINDEXSCALEDI2L:
2824 case INDINDEXSCALEDI2LN:
2825 scale = Address::sxtw(size);
2826 break;
2827 default:
2828 scale = Address::lsl(size);
2829 }
2830
2831 if (index == -1) {
2832 // Fix up any out-of-range offsets.
2833 assert_different_registers(rscratch1, base);
2834 Address addr = Address(base, disp);
2835 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2836 (masm->*insn)(reg, addr);
2837 } else {
2838 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2839 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2840 }
2841 }
2842
2843 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2844 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2845 int opcode, Register base, int index, int size, int disp)
2846 {
2847 if (index == -1) {
2848 (masm->*insn)(reg, T, Address(base, disp));
2849 } else {
2850 assert(disp == 0, "unsupported address mode");
2851 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2852 }
2853 }
2854
2855 %}
2856
2857
2858
2859 //----------ENCODING BLOCK-----------------------------------------------------
2860 // This block specifies the encoding classes used by the compiler to
2861 // output byte streams. Encoding classes are parameterized macros
2862 // used by Machine Instruction Nodes in order to generate the bit
2863 // encoding of the instruction. Operands specify their base encoding
2864 // interface with the interface keyword. There are currently
2865 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2866 // COND_INTER. REG_INTER causes an operand to generate a function
2867 // which returns its register number when queried. CONST_INTER causes
2868 // an operand to generate a function which returns the value of the
2869 // constant when queried. MEMORY_INTER causes an operand to generate
2870 // four functions which return the Base Register, the Index Register,
2871 // the Scale Value, and the Offset Value of the operand when queried.
2872 // COND_INTER causes an operand to generate six functions which return
2873 // the encoding code (ie - encoding bits for the instruction)
2874 // associated with each basic boolean condition for a conditional
2875 // instruction.
2876 //
2877 // Instructions specify two basic values for encoding. Again, a
2878 // function is available to check if the constant displacement is an
2879 // oop. They use the ins_encode keyword to specify their encoding
2880 // classes (which must be a sequence of enc_class names, and their
2881 // parameters, specified in the encoding block), and they use the
2882 // opcode keyword to specify, in order, their primary, secondary, and
2883 // tertiary opcode. Only the opcode sections which a particular
2884 // instruction needs for encoding need to be specified.
2885 encode %{
2886 // Build emit functions for each basic byte or larger field in the
2887 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2888 // from C++ code in the enc_class source block. Emit functions will
2889 // live in the main source block for now. In future, we can
2890 // generalize this by adding a syntax that specifies the sizes of
2891 // fields in an order, so that the adlc can build the emit functions
2892 // automagically
2893
2894 // catch all for unimplemented encodings
2895 enc_class enc_unimplemented %{
2896 __ unimplemented("C2 catch all");
2897 %}
2898
2899 // BEGIN Non-volatile memory access
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2968 Register dst_reg = as_Register($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2976 Register dst_reg = as_Register($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2984 Register dst_reg = as_Register($dst$$reg);
2985 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2992 Register dst_reg = as_Register($dst$$reg);
2993 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
3000 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3001 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3008 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3009 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3010 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3011 %}
3012
3013 // This encoding class is generated automatically from ad_encode.m4.
3014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3015 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3016 Register src_reg = as_Register($src$$reg);
3017 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3018 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3019 %}
3020
3021 // This encoding class is generated automatically from ad_encode.m4.
3022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3023 enc_class aarch64_enc_strb0(memory1 mem) %{
3024 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3025 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3026 %}
3027
3028 // This encoding class is generated automatically from ad_encode.m4.
3029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3030 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3031 Register src_reg = as_Register($src$$reg);
3032 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3033 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3034 %}
3035
3036 // This encoding class is generated automatically from ad_encode.m4.
3037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3038 enc_class aarch64_enc_strh0(memory2 mem) %{
3039 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3041 %}
3042
3043 // This encoding class is generated automatically from ad_encode.m4.
3044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3045 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3046 Register src_reg = as_Register($src$$reg);
3047 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3048 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3049 %}
3050
3051 // This encoding class is generated automatically from ad_encode.m4.
3052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3053 enc_class aarch64_enc_strw0(memory4 mem) %{
3054 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3055 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3056 %}
3057
3058 // This encoding class is generated automatically from ad_encode.m4.
3059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3060 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3061 Register src_reg = as_Register($src$$reg);
3062 // we sometimes get asked to store the stack pointer into the
3063 // current thread -- we cannot do that directly on AArch64
3064 if (src_reg == r31_sp) {
3065 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3066 __ mov(rscratch2, sp);
3067 src_reg = rscratch2;
3068 }
3069 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3070 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3071 %}
3072
3073 // This encoding class is generated automatically from ad_encode.m4.
3074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3075 enc_class aarch64_enc_str0(memory8 mem) %{
3076 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3077 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3078 %}
3079
3080 // This encoding class is generated automatically from ad_encode.m4.
3081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3082 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3083 FloatRegister src_reg = as_FloatRegister($src$$reg);
3084 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3085 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3086 %}
3087
3088 // This encoding class is generated automatically from ad_encode.m4.
3089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3090 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3091 FloatRegister src_reg = as_FloatRegister($src$$reg);
3092 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3093 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3094 %}
3095
3096 // This encoding class is generated automatically from ad_encode.m4.
3097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3098 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3099 __ membar(Assembler::StoreStore);
3100 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3101 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3102 %}
3103
3104 // END Non-volatile memory access
3105
3106 // Vector loads and stores
3107 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3108 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3109 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3110 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3111 %}
3112
3113 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3114 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3115 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3116 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3117 %}
3118
3119 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3120 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3121 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3122 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3123 %}
3124
3125 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3126 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3127 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3128 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3129 %}
3130
3131 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3132 FloatRegister src_reg = as_FloatRegister($src$$reg);
3133 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3134 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3135 %}
3136
3137 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3138 FloatRegister src_reg = as_FloatRegister($src$$reg);
3139 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3140 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3141 %}
3142
3143 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3144 FloatRegister src_reg = as_FloatRegister($src$$reg);
3145 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3146 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3147 %}
3148
3149 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3150 FloatRegister src_reg = as_FloatRegister($src$$reg);
3151 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3152 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3153 %}
3154
3155 // volatile loads and stores
3156
3157 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3158 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, stlrb);
3160 %}
3161
3162 enc_class aarch64_enc_stlrb0(memory mem) %{
3163 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, stlrb);
3165 %}
3166
3167 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3168 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, stlrh);
3170 %}
3171
3172 enc_class aarch64_enc_stlrh0(memory mem) %{
3173 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, stlrh);
3175 %}
3176
3177 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3178 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, stlrw);
3180 %}
3181
3182 enc_class aarch64_enc_stlrw0(memory mem) %{
3183 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, stlrw);
3185 %}
3186
3187 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3188 Register dst_reg = as_Register($dst$$reg);
3189 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3190 rscratch1, ldarb);
3191 __ sxtbw(dst_reg, dst_reg);
3192 %}
3193
3194 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3195 Register dst_reg = as_Register($dst$$reg);
3196 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3197 rscratch1, ldarb);
3198 __ sxtb(dst_reg, dst_reg);
3199 %}
3200
3201 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3202 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, ldarb);
3204 %}
3205
3206 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3207 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3208 rscratch1, ldarb);
3209 %}
3210
3211 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3212 Register dst_reg = as_Register($dst$$reg);
3213 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3214 rscratch1, ldarh);
3215 __ sxthw(dst_reg, dst_reg);
3216 %}
3217
3218 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3219 Register dst_reg = as_Register($dst$$reg);
3220 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldarh);
3222 __ sxth(dst_reg, dst_reg);
3223 %}
3224
3225 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3226 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, ldarh);
3228 %}
3229
3230 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3231 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldarh);
3233 %}
3234
3235 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3236 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, ldarw);
3238 %}
3239
3240 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3241 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldarw);
3243 %}
3244
3245 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3246 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, ldar);
3248 %}
3249
3250 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3251 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3252 rscratch1, ldarw);
3253 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3254 %}
3255
3256 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3257 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3258 rscratch1, ldar);
3259 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3260 %}
3261
3262 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3263 Register src_reg = as_Register($src$$reg);
3264 // we sometimes get asked to store the stack pointer into the
3265 // current thread -- we cannot do that directly on AArch64
3266 if (src_reg == r31_sp) {
3267 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3268 __ mov(rscratch2, sp);
3269 src_reg = rscratch2;
3270 }
3271 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3272 rscratch1, stlr);
3273 %}
3274
3275 enc_class aarch64_enc_stlr0(memory mem) %{
3276 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, stlr);
3278 %}
3279
3280 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3281 {
3282 FloatRegister src_reg = as_FloatRegister($src$$reg);
3283 __ fmovs(rscratch2, src_reg);
3284 }
3285 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3286 rscratch1, stlrw);
3287 %}
3288
3289 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3290 {
3291 FloatRegister src_reg = as_FloatRegister($src$$reg);
3292 __ fmovd(rscratch2, src_reg);
3293 }
3294 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3295 rscratch1, stlr);
3296 %}
3297
3298 // synchronized read/update encodings
3299
3300 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3301 Register dst_reg = as_Register($dst$$reg);
3302 Register base = as_Register($mem$$base);
3303 int index = $mem$$index;
3304 int scale = $mem$$scale;
3305 int disp = $mem$$disp;
3306 if (index == -1) {
3307 if (disp != 0) {
3308 __ lea(rscratch1, Address(base, disp));
3309 __ ldaxr(dst_reg, rscratch1);
3310 } else {
3311 // TODO
3312 // should we ever get anything other than this case?
3313 __ ldaxr(dst_reg, base);
3314 }
3315 } else {
3316 Register index_reg = as_Register(index);
3317 if (disp == 0) {
3318 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3319 __ ldaxr(dst_reg, rscratch1);
3320 } else {
3321 __ lea(rscratch1, Address(base, disp));
3322 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3323 __ ldaxr(dst_reg, rscratch1);
3324 }
3325 }
3326 %}
3327
3328 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3329 Register src_reg = as_Register($src$$reg);
3330 Register base = as_Register($mem$$base);
3331 int index = $mem$$index;
3332 int scale = $mem$$scale;
3333 int disp = $mem$$disp;
3334 if (index == -1) {
3335 if (disp != 0) {
3336 __ lea(rscratch2, Address(base, disp));
3337 __ stlxr(rscratch1, src_reg, rscratch2);
3338 } else {
3339 // TODO
3340 // should we ever get anything other than this case?
3341 __ stlxr(rscratch1, src_reg, base);
3342 }
3343 } else {
3344 Register index_reg = as_Register(index);
3345 if (disp == 0) {
3346 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3347 __ stlxr(rscratch1, src_reg, rscratch2);
3348 } else {
3349 __ lea(rscratch2, Address(base, disp));
3350 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3351 __ stlxr(rscratch1, src_reg, rscratch2);
3352 }
3353 }
3354 __ cmpw(rscratch1, zr);
3355 %}
3356
3357 // prefetch encodings
3358
3359 enc_class aarch64_enc_prefetchw(memory mem) %{
3360 Register base = as_Register($mem$$base);
3361 int index = $mem$$index;
3362 int scale = $mem$$scale;
3363 int disp = $mem$$disp;
3364 if (index == -1) {
3365 // Fix up any out-of-range offsets.
3366 assert_different_registers(rscratch1, base);
3367 Address addr = Address(base, disp);
3368 addr = __ legitimize_address(addr, 8, rscratch1);
3369 __ prfm(addr, PSTL1KEEP);
3370 } else {
3371 Register index_reg = as_Register(index);
3372 if (disp == 0) {
3373 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3374 } else {
3375 __ lea(rscratch1, Address(base, disp));
3376 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3377 }
3378 }
3379 %}
3380
3381 // mov encodings
3382
3383 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3384 uint32_t con = (uint32_t)$src$$constant;
3385 Register dst_reg = as_Register($dst$$reg);
3386 if (con == 0) {
3387 __ movw(dst_reg, zr);
3388 } else {
3389 __ movw(dst_reg, con);
3390 }
3391 %}
3392
3393 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3394 Register dst_reg = as_Register($dst$$reg);
3395 uint64_t con = (uint64_t)$src$$constant;
3396 if (con == 0) {
3397 __ mov(dst_reg, zr);
3398 } else {
3399 __ mov(dst_reg, con);
3400 }
3401 %}
3402
3403 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3404 Register dst_reg = as_Register($dst$$reg);
3405 address con = (address)$src$$constant;
3406 if (con == nullptr || con == (address)1) {
3407 ShouldNotReachHere();
3408 } else {
3409 relocInfo::relocType rtype = $src->constant_reloc();
3410 if (rtype == relocInfo::oop_type) {
3411 __ movoop(dst_reg, (jobject)con);
3412 } else if (rtype == relocInfo::metadata_type) {
3413 __ mov_metadata(dst_reg, (Metadata*)con);
3414 } else {
3415 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3416 // load fake address constants using a normal move
3417 if (! __ is_valid_AArch64_address(con) ||
3418 con < (address)(uintptr_t)os::vm_page_size() ||
3419 rtype == relocInfo::none) {
3420 __ mov(dst_reg, con);
3421 } else {
3422 // use shorter adrp/add sequence for external_word relocation
3423 uint64_t offset;
3424 __ adrp(dst_reg, Address(con, rtype), offset);
3425 __ add(dst_reg, dst_reg, offset);
3426 }
3427 }
3428 }
3429 %}
3430
3431 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3432 Register dst_reg = as_Register($dst$$reg);
3433 __ mov(dst_reg, zr);
3434 %}
3435
3436 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3437 Register dst_reg = as_Register($dst$$reg);
3438 __ mov(dst_reg, (uint64_t)1);
3439 %}
3440
3441 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3442 Register dst_reg = as_Register($dst$$reg);
3443 address con = (address)$src$$constant;
3444 if (con == nullptr) {
3445 ShouldNotReachHere();
3446 } else {
3447 relocInfo::relocType rtype = $src->constant_reloc();
3448 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3449 __ set_narrow_oop(dst_reg, (jobject)con);
3450 }
3451 %}
3452
3453 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3454 Register dst_reg = as_Register($dst$$reg);
3455 __ mov(dst_reg, zr);
3456 %}
3457
3458 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3459 Register dst_reg = as_Register($dst$$reg);
3460 address con = (address)$src$$constant;
3461 if (con == nullptr) {
3462 ShouldNotReachHere();
3463 } else {
3464 relocInfo::relocType rtype = $src->constant_reloc();
3465 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3466 __ set_narrow_klass(dst_reg, (Klass *)con);
3467 }
3468 %}
3469
3470 // arithmetic encodings
3471
3472 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3473 Register dst_reg = as_Register($dst$$reg);
3474 Register src_reg = as_Register($src1$$reg);
3475 int32_t con = (int32_t)$src2$$constant;
3476 // add has primary == 0, subtract has primary == 1
3477 if ($primary) { con = -con; }
3478 if (con < 0) {
3479 __ subw(dst_reg, src_reg, -con);
3480 } else {
3481 __ addw(dst_reg, src_reg, con);
3482 }
3483 %}
3484
3485 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3486 Register dst_reg = as_Register($dst$$reg);
3487 Register src_reg = as_Register($src1$$reg);
3488 int32_t con = (int32_t)$src2$$constant;
3489 // add has primary == 0, subtract has primary == 1
3490 if ($primary) { con = -con; }
3491 if (con < 0) {
3492 __ sub(dst_reg, src_reg, -con);
3493 } else {
3494 __ add(dst_reg, src_reg, con);
3495 }
3496 %}
3497
3498 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3499 Register dst_reg = as_Register($dst$$reg);
3500 Register src1_reg = as_Register($src1$$reg);
3501 Register src2_reg = as_Register($src2$$reg);
3502 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3503 %}
3504
3505 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3506 Register dst_reg = as_Register($dst$$reg);
3507 Register src1_reg = as_Register($src1$$reg);
3508 Register src2_reg = as_Register($src2$$reg);
3509 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3510 %}
3511
3512 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3513 Register dst_reg = as_Register($dst$$reg);
3514 Register src1_reg = as_Register($src1$$reg);
3515 Register src2_reg = as_Register($src2$$reg);
3516 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3517 %}
3518
3519 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3520 Register dst_reg = as_Register($dst$$reg);
3521 Register src1_reg = as_Register($src1$$reg);
3522 Register src2_reg = as_Register($src2$$reg);
3523 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3524 %}
3525
3526 // compare instruction encodings
3527
3528 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3529 Register reg1 = as_Register($src1$$reg);
3530 Register reg2 = as_Register($src2$$reg);
3531 __ cmpw(reg1, reg2);
3532 %}
3533
3534 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3535 Register reg = as_Register($src1$$reg);
3536 int32_t val = $src2$$constant;
3537 if (val >= 0) {
3538 __ subsw(zr, reg, val);
3539 } else {
3540 __ addsw(zr, reg, -val);
3541 }
3542 %}
3543
3544 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3545 Register reg1 = as_Register($src1$$reg);
3546 uint32_t val = (uint32_t)$src2$$constant;
3547 __ movw(rscratch1, val);
3548 __ cmpw(reg1, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3552 Register reg1 = as_Register($src1$$reg);
3553 Register reg2 = as_Register($src2$$reg);
3554 __ cmp(reg1, reg2);
3555 %}
3556
3557 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3558 Register reg = as_Register($src1$$reg);
3559 int64_t val = $src2$$constant;
3560 if (val >= 0) {
3561 __ subs(zr, reg, val);
3562 } else if (val != -val) {
3563 __ adds(zr, reg, -val);
3564 } else {
3565 // aargh, Long.MIN_VALUE is a special case
3566 __ orr(rscratch1, zr, (uint64_t)val);
3567 __ subs(zr, reg, rscratch1);
3568 }
3569 %}
3570
3571 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3572 Register reg1 = as_Register($src1$$reg);
3573 uint64_t val = (uint64_t)$src2$$constant;
3574 __ mov(rscratch1, val);
3575 __ cmp(reg1, rscratch1);
3576 %}
3577
3578 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3579 Register reg1 = as_Register($src1$$reg);
3580 Register reg2 = as_Register($src2$$reg);
3581 __ cmp(reg1, reg2);
3582 %}
3583
3584 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3585 Register reg1 = as_Register($src1$$reg);
3586 Register reg2 = as_Register($src2$$reg);
3587 __ cmpw(reg1, reg2);
3588 %}
3589
3590 enc_class aarch64_enc_testp(iRegP src) %{
3591 Register reg = as_Register($src$$reg);
3592 __ cmp(reg, zr);
3593 %}
3594
3595 enc_class aarch64_enc_testn(iRegN src) %{
3596 Register reg = as_Register($src$$reg);
3597 __ cmpw(reg, zr);
3598 %}
3599
3600 enc_class aarch64_enc_b(label lbl) %{
3601 Label *L = $lbl$$label;
3602 __ b(*L);
3603 %}
3604
3605 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3606 Label *L = $lbl$$label;
3607 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3608 %}
3609
3610 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3611 Label *L = $lbl$$label;
3612 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3613 %}
3614
3615 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3616 %{
3617 Register sub_reg = as_Register($sub$$reg);
3618 Register super_reg = as_Register($super$$reg);
3619 Register temp_reg = as_Register($temp$$reg);
3620 Register result_reg = as_Register($result$$reg);
3621
3622 Label miss;
3623 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3624 nullptr, &miss,
3625 /*set_cond_codes:*/ true);
3626 if ($primary) {
3627 __ mov(result_reg, zr);
3628 }
3629 __ bind(miss);
3630 %}
3631
3632 enc_class aarch64_enc_java_static_call(method meth) %{
3633 address addr = (address)$meth$$method;
3634 address call;
3635 if (!_method) {
3636 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3637 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3638 if (call == nullptr) {
3639 ciEnv::current()->record_failure("CodeCache is full");
3640 return;
3641 }
3642 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3643 // The NOP here is purely to ensure that eliding a call to
3644 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3645 __ nop();
3646 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3647 } else {
3648 int method_index = resolved_method_index(masm);
3649 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3650 : static_call_Relocation::spec(method_index);
3651 call = __ trampoline_call(Address(addr, rspec));
3652 if (call == nullptr) {
3653 ciEnv::current()->record_failure("CodeCache is full");
3654 return;
3655 }
3656 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3657 // Calls of the same statically bound method can share
3658 // a stub to the interpreter.
3659 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3660 } else {
3661 // Emit stub for static call
3662 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3663 if (stub == nullptr) {
3664 ciEnv::current()->record_failure("CodeCache is full");
3665 return;
3666 }
3667 }
3668 }
3669
3670 __ post_call_nop();
3671
3672 // Only non uncommon_trap calls need to reinitialize ptrue.
3673 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3674 __ reinitialize_ptrue();
3675 }
3676 %}
3677
3678 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3679 int method_index = resolved_method_index(masm);
3680 address call = __ ic_call((address)$meth$$method, method_index);
3681 if (call == nullptr) {
3682 ciEnv::current()->record_failure("CodeCache is full");
3683 return;
3684 }
3685 __ post_call_nop();
3686 if (Compile::current()->max_vector_size() > 0) {
3687 __ reinitialize_ptrue();
3688 }
3689 %}
3690
3691 enc_class aarch64_enc_call_epilog() %{
3692 if (VerifyStackAtCalls) {
3693 // Check that stack depth is unchanged: find majik cookie on stack
3694 __ call_Unimplemented();
3695 }
3696 %}
3697
3698 enc_class aarch64_enc_java_to_runtime(method meth) %{
3699 // some calls to generated routines (arraycopy code) are scheduled
3700 // by C2 as runtime calls. if so we can call them using a br (they
3701 // will be in a reachable segment) otherwise we have to use a blr
3702 // which loads the absolute address into a register.
3703 address entry = (address)$meth$$method;
3704 CodeBlob *cb = CodeCache::find_blob(entry);
3705 if (cb) {
3706 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3707 if (call == nullptr) {
3708 ciEnv::current()->record_failure("CodeCache is full");
3709 return;
3710 }
3711 __ post_call_nop();
3712 } else {
3713 Label retaddr;
3714 // Make the anchor frame walkable
3715 __ adr(rscratch2, retaddr);
3716 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3717 __ lea(rscratch1, RuntimeAddress(entry));
3718 __ blr(rscratch1);
3719 __ bind(retaddr);
3720 __ post_call_nop();
3721 }
3722 if (Compile::current()->max_vector_size() > 0) {
3723 __ reinitialize_ptrue();
3724 }
3725 %}
3726
3727 enc_class aarch64_enc_rethrow() %{
3728 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3729 %}
3730
3731 enc_class aarch64_enc_ret() %{
3732 #ifdef ASSERT
3733 if (Compile::current()->max_vector_size() > 0) {
3734 __ verify_ptrue();
3735 }
3736 #endif
3737 __ ret(lr);
3738 %}
3739
3740 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3741 Register target_reg = as_Register($jump_target$$reg);
3742 __ br(target_reg);
3743 %}
3744
3745 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3746 Register target_reg = as_Register($jump_target$$reg);
3747 // exception oop should be in r0
3748 // ret addr has been popped into lr
3749 // callee expects it in r3
3750 __ mov(r3, lr);
3751 __ br(target_reg);
3752 %}
3753
3754 %}
3755
3756 //----------FRAME--------------------------------------------------------------
3757 // Definition of frame structure and management information.
3758 //
3759 // S T A C K L A Y O U T Allocators stack-slot number
3760 // | (to get allocators register number
3761 // G Owned by | | v add OptoReg::stack0())
3762 // r CALLER | |
3763 // o | +--------+ pad to even-align allocators stack-slot
3764 // w V | pad0 | numbers; owned by CALLER
3765 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3766 // h ^ | in | 5
3767 // | | args | 4 Holes in incoming args owned by SELF
3768 // | | | | 3
3769 // | | +--------+
3770 // V | | old out| Empty on Intel, window on Sparc
3771 // | old |preserve| Must be even aligned.
3772 // | SP-+--------+----> Matcher::_old_SP, even aligned
3773 // | | in | 3 area for Intel ret address
3774 // Owned by |preserve| Empty on Sparc.
3775 // SELF +--------+
3776 // | | pad2 | 2 pad to align old SP
3777 // | +--------+ 1
3778 // | | locks | 0
3779 // | +--------+----> OptoReg::stack0(), even aligned
3780 // | | pad1 | 11 pad to align new SP
3781 // | +--------+
3782 // | | | 10
3783 // | | spills | 9 spills
3784 // V | | 8 (pad0 slot for callee)
3785 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3786 // ^ | out | 7
3787 // | | args | 6 Holes in outgoing args owned by CALLEE
3788 // Owned by +--------+
3789 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3790 // | new |preserve| Must be even-aligned.
3791 // | SP-+--------+----> Matcher::_new_SP, even aligned
3792 // | | |
3793 //
3794 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3795 // known from SELF's arguments and the Java calling convention.
3796 // Region 6-7 is determined per call site.
3797 // Note 2: If the calling convention leaves holes in the incoming argument
3798 // area, those holes are owned by SELF. Holes in the outgoing area
3799 // are owned by the CALLEE. Holes should not be necessary in the
3800 // incoming area, as the Java calling convention is completely under
3801 // the control of the AD file. Doubles can be sorted and packed to
3802 // avoid holes. Holes in the outgoing arguments may be necessary for
3803 // varargs C calling conventions.
3804 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3805 // even aligned with pad0 as needed.
3806 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3807 // (the latter is true on Intel but is it false on AArch64?)
3808 // region 6-11 is even aligned; it may be padded out more so that
3809 // the region from SP to FP meets the minimum stack alignment.
3810 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3811 // alignment. Region 11, pad1, may be dynamically extended so that
3812 // SP meets the minimum alignment.
3813
3814 frame %{
3815 // These three registers define part of the calling convention
3816 // between compiled code and the interpreter.
3817
3818 // Inline Cache Register or Method for I2C.
3819 inline_cache_reg(R12);
3820
3821 // Number of stack slots consumed by locking an object
3822 sync_stack_slots(2);
3823
3824 // Compiled code's Frame Pointer
3825 frame_pointer(R31);
3826
3827 // Stack alignment requirement
3828 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3829
3830 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3831 // for calls to C. Supports the var-args backing area for register parms.
3832 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3833
3834 // The after-PROLOG location of the return address. Location of
3835 // return address specifies a type (REG or STACK) and a number
3836 // representing the register number (i.e. - use a register name) or
3837 // stack slot.
3838 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3839 // Otherwise, it is above the locks and verification slot and alignment word
3840 // TODO this may well be correct but need to check why that - 2 is there
3841 // ppc port uses 0 but we definitely need to allow for fixed_slots
3842 // which folds in the space used for monitors
3843 return_addr(STACK - 2 +
3844 align_up((Compile::current()->in_preserve_stack_slots() +
3845 Compile::current()->fixed_slots()),
3846 stack_alignment_in_slots()));
3847
3848 // Location of compiled Java return values. Same as C for now.
3849 return_value
3850 %{
3851 // TODO do we allow ideal_reg == Op_RegN???
3852 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3853 "only return normal values");
3854
3855 static const int lo[Op_RegL + 1] = { // enum name
3856 0, // Op_Node
3857 0, // Op_Set
3858 R0_num, // Op_RegN
3859 R0_num, // Op_RegI
3860 R0_num, // Op_RegP
3861 V0_num, // Op_RegF
3862 V0_num, // Op_RegD
3863 R0_num // Op_RegL
3864 };
3865
3866 static const int hi[Op_RegL + 1] = { // enum name
3867 0, // Op_Node
3868 0, // Op_Set
3869 OptoReg::Bad, // Op_RegN
3870 OptoReg::Bad, // Op_RegI
3871 R0_H_num, // Op_RegP
3872 OptoReg::Bad, // Op_RegF
3873 V0_H_num, // Op_RegD
3874 R0_H_num // Op_RegL
3875 };
3876
3877 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3878 %}
3879 %}
3880
3881 //----------ATTRIBUTES---------------------------------------------------------
3882 //----------Operand Attributes-------------------------------------------------
3883 op_attrib op_cost(1); // Required cost attribute
3884
3885 //----------Instruction Attributes---------------------------------------------
3886 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3887 ins_attrib ins_size(32); // Required size attribute (in bits)
3888 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3889 // a non-matching short branch variant
3890 // of some long branch?
3891 ins_attrib ins_alignment(4); // Required alignment attribute (must
3892 // be a power of 2) specifies the
3893 // alignment that some part of the
3894 // instruction (not necessarily the
3895 // start) requires. If > 1, a
3896 // compute_padding() function must be
3897 // provided for the instruction
3898
3899 // Whether this node is expanded during code emission into a sequence of
3900 // instructions and the first instruction can perform an implicit null check.
3901 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3902
3903 //----------OPERANDS-----------------------------------------------------------
3904 // Operand definitions must precede instruction definitions for correct parsing
3905 // in the ADLC because operands constitute user defined types which are used in
3906 // instruction definitions.
3907
3908 //----------Simple Operands----------------------------------------------------
3909
3910 // Integer operands 32 bit
3911 // 32 bit immediate
3912 operand immI()
3913 %{
3914 match(ConI);
3915
3916 op_cost(0);
3917 format %{ %}
3918 interface(CONST_INTER);
3919 %}
3920
3921 // 32 bit zero
3922 operand immI0()
3923 %{
3924 predicate(n->get_int() == 0);
3925 match(ConI);
3926
3927 op_cost(0);
3928 format %{ %}
3929 interface(CONST_INTER);
3930 %}
3931
3932 // 32 bit unit increment
3933 operand immI_1()
3934 %{
3935 predicate(n->get_int() == 1);
3936 match(ConI);
3937
3938 op_cost(0);
3939 format %{ %}
3940 interface(CONST_INTER);
3941 %}
3942
3943 // 32 bit unit decrement
3944 operand immI_M1()
3945 %{
3946 predicate(n->get_int() == -1);
3947 match(ConI);
3948
3949 op_cost(0);
3950 format %{ %}
3951 interface(CONST_INTER);
3952 %}
3953
3954 // Shift values for add/sub extension shift
3955 operand immIExt()
3956 %{
3957 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3958 match(ConI);
3959
3960 op_cost(0);
3961 format %{ %}
3962 interface(CONST_INTER);
3963 %}
3964
3965 operand immI_gt_1()
3966 %{
3967 predicate(n->get_int() > 1);
3968 match(ConI);
3969
3970 op_cost(0);
3971 format %{ %}
3972 interface(CONST_INTER);
3973 %}
3974
3975 operand immI_le_4()
3976 %{
3977 predicate(n->get_int() <= 4);
3978 match(ConI);
3979
3980 op_cost(0);
3981 format %{ %}
3982 interface(CONST_INTER);
3983 %}
3984
3985 operand immI_16()
3986 %{
3987 predicate(n->get_int() == 16);
3988 match(ConI);
3989
3990 op_cost(0);
3991 format %{ %}
3992 interface(CONST_INTER);
3993 %}
3994
3995 operand immI_24()
3996 %{
3997 predicate(n->get_int() == 24);
3998 match(ConI);
3999
4000 op_cost(0);
4001 format %{ %}
4002 interface(CONST_INTER);
4003 %}
4004
4005 operand immI_32()
4006 %{
4007 predicate(n->get_int() == 32);
4008 match(ConI);
4009
4010 op_cost(0);
4011 format %{ %}
4012 interface(CONST_INTER);
4013 %}
4014
4015 operand immI_48()
4016 %{
4017 predicate(n->get_int() == 48);
4018 match(ConI);
4019
4020 op_cost(0);
4021 format %{ %}
4022 interface(CONST_INTER);
4023 %}
4024
4025 operand immI_56()
4026 %{
4027 predicate(n->get_int() == 56);
4028 match(ConI);
4029
4030 op_cost(0);
4031 format %{ %}
4032 interface(CONST_INTER);
4033 %}
4034
4035 operand immI_255()
4036 %{
4037 predicate(n->get_int() == 255);
4038 match(ConI);
4039
4040 op_cost(0);
4041 format %{ %}
4042 interface(CONST_INTER);
4043 %}
4044
4045 operand immI_65535()
4046 %{
4047 predicate(n->get_int() == 65535);
4048 match(ConI);
4049
4050 op_cost(0);
4051 format %{ %}
4052 interface(CONST_INTER);
4053 %}
4054
4055 operand immI_positive()
4056 %{
4057 predicate(n->get_int() > 0);
4058 match(ConI);
4059
4060 op_cost(0);
4061 format %{ %}
4062 interface(CONST_INTER);
4063 %}
4064
4065 // BoolTest condition for signed compare
4066 operand immI_cmp_cond()
4067 %{
4068 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4069 match(ConI);
4070
4071 op_cost(0);
4072 format %{ %}
4073 interface(CONST_INTER);
4074 %}
4075
4076 // BoolTest condition for unsigned compare
4077 operand immI_cmpU_cond()
4078 %{
4079 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4080 match(ConI);
4081
4082 op_cost(0);
4083 format %{ %}
4084 interface(CONST_INTER);
4085 %}
4086
4087 operand immL_255()
4088 %{
4089 predicate(n->get_long() == 255L);
4090 match(ConL);
4091
4092 op_cost(0);
4093 format %{ %}
4094 interface(CONST_INTER);
4095 %}
4096
4097 operand immL_65535()
4098 %{
4099 predicate(n->get_long() == 65535L);
4100 match(ConL);
4101
4102 op_cost(0);
4103 format %{ %}
4104 interface(CONST_INTER);
4105 %}
4106
4107 operand immL_4294967295()
4108 %{
4109 predicate(n->get_long() == 4294967295L);
4110 match(ConL);
4111
4112 op_cost(0);
4113 format %{ %}
4114 interface(CONST_INTER);
4115 %}
4116
4117 operand immL_bitmask()
4118 %{
4119 predicate((n->get_long() != 0)
4120 && ((n->get_long() & 0xc000000000000000l) == 0)
4121 && is_power_of_2(n->get_long() + 1));
4122 match(ConL);
4123
4124 op_cost(0);
4125 format %{ %}
4126 interface(CONST_INTER);
4127 %}
4128
4129 operand immI_bitmask()
4130 %{
4131 predicate((n->get_int() != 0)
4132 && ((n->get_int() & 0xc0000000) == 0)
4133 && is_power_of_2(n->get_int() + 1));
4134 match(ConI);
4135
4136 op_cost(0);
4137 format %{ %}
4138 interface(CONST_INTER);
4139 %}
4140
4141 operand immL_positive_bitmaskI()
4142 %{
4143 predicate((n->get_long() != 0)
4144 && ((julong)n->get_long() < 0x80000000ULL)
4145 && is_power_of_2(n->get_long() + 1));
4146 match(ConL);
4147
4148 op_cost(0);
4149 format %{ %}
4150 interface(CONST_INTER);
4151 %}
4152
4153 // Scale values for scaled offset addressing modes (up to long but not quad)
4154 operand immIScale()
4155 %{
4156 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4157 match(ConI);
4158
4159 op_cost(0);
4160 format %{ %}
4161 interface(CONST_INTER);
4162 %}
4163
4164 // 5 bit signed integer
4165 operand immI5()
4166 %{
4167 predicate(Assembler::is_simm(n->get_int(), 5));
4168 match(ConI);
4169
4170 op_cost(0);
4171 format %{ %}
4172 interface(CONST_INTER);
4173 %}
4174
4175 // 7 bit unsigned integer
4176 operand immIU7()
4177 %{
4178 predicate(Assembler::is_uimm(n->get_int(), 7));
4179 match(ConI);
4180
4181 op_cost(0);
4182 format %{ %}
4183 interface(CONST_INTER);
4184 %}
4185
4186 // Offset for scaled or unscaled immediate loads and stores
4187 operand immIOffset()
4188 %{
4189 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4190 match(ConI);
4191
4192 op_cost(0);
4193 format %{ %}
4194 interface(CONST_INTER);
4195 %}
4196
4197 operand immIOffset1()
4198 %{
4199 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4200 match(ConI);
4201
4202 op_cost(0);
4203 format %{ %}
4204 interface(CONST_INTER);
4205 %}
4206
4207 operand immIOffset2()
4208 %{
4209 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4210 match(ConI);
4211
4212 op_cost(0);
4213 format %{ %}
4214 interface(CONST_INTER);
4215 %}
4216
4217 operand immIOffset4()
4218 %{
4219 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4220 match(ConI);
4221
4222 op_cost(0);
4223 format %{ %}
4224 interface(CONST_INTER);
4225 %}
4226
4227 operand immIOffset8()
4228 %{
4229 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4230 match(ConI);
4231
4232 op_cost(0);
4233 format %{ %}
4234 interface(CONST_INTER);
4235 %}
4236
4237 operand immIOffset16()
4238 %{
4239 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4240 match(ConI);
4241
4242 op_cost(0);
4243 format %{ %}
4244 interface(CONST_INTER);
4245 %}
4246
4247 operand immLOffset()
4248 %{
4249 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4250 match(ConL);
4251
4252 op_cost(0);
4253 format %{ %}
4254 interface(CONST_INTER);
4255 %}
4256
4257 operand immLoffset1()
4258 %{
4259 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4260 match(ConL);
4261
4262 op_cost(0);
4263 format %{ %}
4264 interface(CONST_INTER);
4265 %}
4266
4267 operand immLoffset2()
4268 %{
4269 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4270 match(ConL);
4271
4272 op_cost(0);
4273 format %{ %}
4274 interface(CONST_INTER);
4275 %}
4276
4277 operand immLoffset4()
4278 %{
4279 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4280 match(ConL);
4281
4282 op_cost(0);
4283 format %{ %}
4284 interface(CONST_INTER);
4285 %}
4286
4287 operand immLoffset8()
4288 %{
4289 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4290 match(ConL);
4291
4292 op_cost(0);
4293 format %{ %}
4294 interface(CONST_INTER);
4295 %}
4296
4297 operand immLoffset16()
4298 %{
4299 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4300 match(ConL);
4301
4302 op_cost(0);
4303 format %{ %}
4304 interface(CONST_INTER);
4305 %}
4306
4307 // 5 bit signed long integer
4308 operand immL5()
4309 %{
4310 predicate(Assembler::is_simm(n->get_long(), 5));
4311 match(ConL);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 // 7 bit unsigned long integer
4319 operand immLU7()
4320 %{
4321 predicate(Assembler::is_uimm(n->get_long(), 7));
4322 match(ConL);
4323
4324 op_cost(0);
4325 format %{ %}
4326 interface(CONST_INTER);
4327 %}
4328
4329 // 8 bit signed value.
4330 operand immI8()
4331 %{
4332 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4333 match(ConI);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 // 8 bit signed value (simm8), or #simm8 LSL 8.
4341 operand immIDupV()
4342 %{
4343 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4344 match(ConI);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 // 8 bit signed value (simm8), or #simm8 LSL 8.
4352 operand immLDupV()
4353 %{
4354 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4355 match(ConL);
4356
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 // 8 bit signed value (simm8), or #simm8 LSL 8.
4363 operand immHDupV()
4364 %{
4365 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4366 match(ConH);
4367
4368 op_cost(0);
4369 format %{ %}
4370 interface(CONST_INTER);
4371 %}
4372
4373 // 8 bit integer valid for vector add sub immediate
4374 operand immBAddSubV()
4375 %{
4376 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4377 match(ConI);
4378
4379 op_cost(0);
4380 format %{ %}
4381 interface(CONST_INTER);
4382 %}
4383
4384 // 32 bit integer valid for add sub immediate
4385 operand immIAddSub()
4386 %{
4387 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4388 match(ConI);
4389 op_cost(0);
4390 format %{ %}
4391 interface(CONST_INTER);
4392 %}
4393
4394 // 32 bit integer valid for vector add sub immediate
4395 operand immIAddSubV()
4396 %{
4397 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4398 match(ConI);
4399
4400 op_cost(0);
4401 format %{ %}
4402 interface(CONST_INTER);
4403 %}
4404
4405 // 32 bit unsigned integer valid for logical immediate
4406
4407 operand immBLog()
4408 %{
4409 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4410 match(ConI);
4411
4412 op_cost(0);
4413 format %{ %}
4414 interface(CONST_INTER);
4415 %}
4416
4417 operand immSLog()
4418 %{
4419 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4420 match(ConI);
4421
4422 op_cost(0);
4423 format %{ %}
4424 interface(CONST_INTER);
4425 %}
4426
4427 operand immILog()
4428 %{
4429 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4430 match(ConI);
4431
4432 op_cost(0);
4433 format %{ %}
4434 interface(CONST_INTER);
4435 %}
4436
4437 // Integer operands 64 bit
4438 // 64 bit immediate
4439 operand immL()
4440 %{
4441 match(ConL);
4442
4443 op_cost(0);
4444 format %{ %}
4445 interface(CONST_INTER);
4446 %}
4447
4448 // 64 bit zero
4449 operand immL0()
4450 %{
4451 predicate(n->get_long() == 0);
4452 match(ConL);
4453
4454 op_cost(0);
4455 format %{ %}
4456 interface(CONST_INTER);
4457 %}
4458
4459 // 64 bit unit decrement
4460 operand immL_M1()
4461 %{
4462 predicate(n->get_long() == -1);
4463 match(ConL);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 // 64 bit integer valid for add sub immediate
4471 operand immLAddSub()
4472 %{
4473 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4474 match(ConL);
4475 op_cost(0);
4476 format %{ %}
4477 interface(CONST_INTER);
4478 %}
4479
4480 // 64 bit integer valid for addv subv immediate
4481 operand immLAddSubV()
4482 %{
4483 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4484 match(ConL);
4485
4486 op_cost(0);
4487 format %{ %}
4488 interface(CONST_INTER);
4489 %}
4490
4491 // 64 bit integer valid for logical immediate
4492 operand immLLog()
4493 %{
4494 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4495 match(ConL);
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // Long Immediate: low 32-bit mask
4502 operand immL_32bits()
4503 %{
4504 predicate(n->get_long() == 0xFFFFFFFFL);
4505 match(ConL);
4506 op_cost(0);
4507 format %{ %}
4508 interface(CONST_INTER);
4509 %}
4510
4511 // Pointer operands
4512 // Pointer Immediate
4513 operand immP()
4514 %{
4515 match(ConP);
4516
4517 op_cost(0);
4518 format %{ %}
4519 interface(CONST_INTER);
4520 %}
4521
4522 // nullptr Pointer Immediate
4523 operand immP0()
4524 %{
4525 predicate(n->get_ptr() == 0);
4526 match(ConP);
4527
4528 op_cost(0);
4529 format %{ %}
4530 interface(CONST_INTER);
4531 %}
4532
4533 // Pointer Immediate One
4534 // this is used in object initialization (initial object header)
4535 operand immP_1()
4536 %{
4537 predicate(n->get_ptr() == 1);
4538 match(ConP);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // AOT Runtime Constants Address
4546 operand immAOTRuntimeConstantsAddress()
4547 %{
4548 // Check if the address is in the range of AOT Runtime Constants
4549 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4550 match(ConP);
4551
4552 op_cost(0);
4553 format %{ %}
4554 interface(CONST_INTER);
4555 %}
4556
4557 // Float and Double operands
4558 // Double Immediate
4559 operand immD()
4560 %{
4561 match(ConD);
4562 op_cost(0);
4563 format %{ %}
4564 interface(CONST_INTER);
4565 %}
4566
4567 // Double Immediate: +0.0d
4568 operand immD0()
4569 %{
4570 predicate(jlong_cast(n->getd()) == 0);
4571 match(ConD);
4572
4573 op_cost(0);
4574 format %{ %}
4575 interface(CONST_INTER);
4576 %}
4577
4578 // constant 'double +0.0'.
4579 operand immDPacked()
4580 %{
4581 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4582 match(ConD);
4583 op_cost(0);
4584 format %{ %}
4585 interface(CONST_INTER);
4586 %}
4587
4588 // Float Immediate
4589 operand immF()
4590 %{
4591 match(ConF);
4592 op_cost(0);
4593 format %{ %}
4594 interface(CONST_INTER);
4595 %}
4596
4597 // Float Immediate: +0.0f.
4598 operand immF0()
4599 %{
4600 predicate(jint_cast(n->getf()) == 0);
4601 match(ConF);
4602
4603 op_cost(0);
4604 format %{ %}
4605 interface(CONST_INTER);
4606 %}
4607
4608 // Half Float (FP16) Immediate
4609 operand immH()
4610 %{
4611 match(ConH);
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 //
4618 operand immFPacked()
4619 %{
4620 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4621 match(ConF);
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 // Narrow pointer operands
4628 // Narrow Pointer Immediate
4629 operand immN()
4630 %{
4631 match(ConN);
4632
4633 op_cost(0);
4634 format %{ %}
4635 interface(CONST_INTER);
4636 %}
4637
4638 // Narrow nullptr Pointer Immediate
4639 operand immN0()
4640 %{
4641 predicate(n->get_narrowcon() == 0);
4642 match(ConN);
4643
4644 op_cost(0);
4645 format %{ %}
4646 interface(CONST_INTER);
4647 %}
4648
4649 operand immNKlass()
4650 %{
4651 match(ConNKlass);
4652
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 // Integer 32 bit Register Operands
4659 // Integer 32 bitRegister (excludes SP)
4660 operand iRegI()
4661 %{
4662 constraint(ALLOC_IN_RC(any_reg32));
4663 match(RegI);
4664 match(iRegINoSp);
4665 op_cost(0);
4666 format %{ %}
4667 interface(REG_INTER);
4668 %}
4669
4670 // Integer 32 bit Register not Special
4671 operand iRegINoSp()
4672 %{
4673 constraint(ALLOC_IN_RC(no_special_reg32));
4674 match(RegI);
4675 op_cost(0);
4676 format %{ %}
4677 interface(REG_INTER);
4678 %}
4679
4680 // Integer 64 bit Register Operands
4681 // Integer 64 bit Register (includes SP)
4682 operand iRegL()
4683 %{
4684 constraint(ALLOC_IN_RC(any_reg));
4685 match(RegL);
4686 match(iRegLNoSp);
4687 op_cost(0);
4688 format %{ %}
4689 interface(REG_INTER);
4690 %}
4691
4692 // Integer 64 bit Register not Special
4693 operand iRegLNoSp()
4694 %{
4695 constraint(ALLOC_IN_RC(no_special_reg));
4696 match(RegL);
4697 match(iRegL_R0);
4698 format %{ %}
4699 interface(REG_INTER);
4700 %}
4701
4702 // Pointer Register Operands
4703 // Pointer Register
4704 operand iRegP()
4705 %{
4706 constraint(ALLOC_IN_RC(ptr_reg));
4707 match(RegP);
4708 match(iRegPNoSp);
4709 match(iRegP_R0);
4710 //match(iRegP_R2);
4711 //match(iRegP_R4);
4712 match(iRegP_R5);
4713 match(thread_RegP);
4714 op_cost(0);
4715 format %{ %}
4716 interface(REG_INTER);
4717 %}
4718
4719 // Pointer 64 bit Register not Special
4720 operand iRegPNoSp()
4721 %{
4722 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4723 match(RegP);
4724 // match(iRegP);
4725 // match(iRegP_R0);
4726 // match(iRegP_R2);
4727 // match(iRegP_R4);
4728 // match(iRegP_R5);
4729 // match(thread_RegP);
4730 op_cost(0);
4731 format %{ %}
4732 interface(REG_INTER);
4733 %}
4734
4735 // This operand is not allowed to use rfp even if
4736 // rfp is not used to hold the frame pointer.
4737 operand iRegPNoSpNoRfp()
4738 %{
4739 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4740 match(RegP);
4741 match(iRegPNoSp);
4742 op_cost(0);
4743 format %{ %}
4744 interface(REG_INTER);
4745 %}
4746
4747 // Pointer 64 bit Register R0 only
4748 operand iRegP_R0()
4749 %{
4750 constraint(ALLOC_IN_RC(r0_reg));
4751 match(RegP);
4752 // match(iRegP);
4753 match(iRegPNoSp);
4754 op_cost(0);
4755 format %{ %}
4756 interface(REG_INTER);
4757 %}
4758
4759 // Pointer 64 bit Register R1 only
4760 operand iRegP_R1()
4761 %{
4762 constraint(ALLOC_IN_RC(r1_reg));
4763 match(RegP);
4764 // match(iRegP);
4765 match(iRegPNoSp);
4766 op_cost(0);
4767 format %{ %}
4768 interface(REG_INTER);
4769 %}
4770
4771 // Pointer 64 bit Register R2 only
4772 operand iRegP_R2()
4773 %{
4774 constraint(ALLOC_IN_RC(r2_reg));
4775 match(RegP);
4776 // match(iRegP);
4777 match(iRegPNoSp);
4778 op_cost(0);
4779 format %{ %}
4780 interface(REG_INTER);
4781 %}
4782
4783 // Pointer 64 bit Register R3 only
4784 operand iRegP_R3()
4785 %{
4786 constraint(ALLOC_IN_RC(r3_reg));
4787 match(RegP);
4788 // match(iRegP);
4789 match(iRegPNoSp);
4790 op_cost(0);
4791 format %{ %}
4792 interface(REG_INTER);
4793 %}
4794
4795 // Pointer 64 bit Register R4 only
4796 operand iRegP_R4()
4797 %{
4798 constraint(ALLOC_IN_RC(r4_reg));
4799 match(RegP);
4800 // match(iRegP);
4801 match(iRegPNoSp);
4802 op_cost(0);
4803 format %{ %}
4804 interface(REG_INTER);
4805 %}
4806
4807 // Pointer 64 bit Register R5 only
4808 operand iRegP_R5()
4809 %{
4810 constraint(ALLOC_IN_RC(r5_reg));
4811 match(RegP);
4812 // match(iRegP);
4813 match(iRegPNoSp);
4814 op_cost(0);
4815 format %{ %}
4816 interface(REG_INTER);
4817 %}
4818
4819 // Pointer 64 bit Register R10 only
4820 operand iRegP_R10()
4821 %{
4822 constraint(ALLOC_IN_RC(r10_reg));
4823 match(RegP);
4824 // match(iRegP);
4825 match(iRegPNoSp);
4826 op_cost(0);
4827 format %{ %}
4828 interface(REG_INTER);
4829 %}
4830
4831 // Long 64 bit Register R0 only
4832 operand iRegL_R0()
4833 %{
4834 constraint(ALLOC_IN_RC(r0_reg));
4835 match(RegL);
4836 match(iRegLNoSp);
4837 op_cost(0);
4838 format %{ %}
4839 interface(REG_INTER);
4840 %}
4841
4842 // Long 64 bit Register R11 only
4843 operand iRegL_R11()
4844 %{
4845 constraint(ALLOC_IN_RC(r11_reg));
4846 match(RegL);
4847 match(iRegLNoSp);
4848 op_cost(0);
4849 format %{ %}
4850 interface(REG_INTER);
4851 %}
4852
4853 // Register R0 only
4854 operand iRegI_R0()
4855 %{
4856 constraint(ALLOC_IN_RC(int_r0_reg));
4857 match(RegI);
4858 match(iRegINoSp);
4859 op_cost(0);
4860 format %{ %}
4861 interface(REG_INTER);
4862 %}
4863
4864 // Register R2 only
4865 operand iRegI_R2()
4866 %{
4867 constraint(ALLOC_IN_RC(int_r2_reg));
4868 match(RegI);
4869 match(iRegINoSp);
4870 op_cost(0);
4871 format %{ %}
4872 interface(REG_INTER);
4873 %}
4874
4875 // Register R3 only
4876 operand iRegI_R3()
4877 %{
4878 constraint(ALLOC_IN_RC(int_r3_reg));
4879 match(RegI);
4880 match(iRegINoSp);
4881 op_cost(0);
4882 format %{ %}
4883 interface(REG_INTER);
4884 %}
4885
4886
4887 // Register R4 only
4888 operand iRegI_R4()
4889 %{
4890 constraint(ALLOC_IN_RC(int_r4_reg));
4891 match(RegI);
4892 match(iRegINoSp);
4893 op_cost(0);
4894 format %{ %}
4895 interface(REG_INTER);
4896 %}
4897
4898
4899 // Pointer Register Operands
4900 // Narrow Pointer Register
4901 operand iRegN()
4902 %{
4903 constraint(ALLOC_IN_RC(any_reg32));
4904 match(RegN);
4905 match(iRegNNoSp);
4906 op_cost(0);
4907 format %{ %}
4908 interface(REG_INTER);
4909 %}
4910
4911 // Integer 64 bit Register not Special
4912 operand iRegNNoSp()
4913 %{
4914 constraint(ALLOC_IN_RC(no_special_reg32));
4915 match(RegN);
4916 op_cost(0);
4917 format %{ %}
4918 interface(REG_INTER);
4919 %}
4920
4921 // Float Register
4922 // Float register operands
4923 operand vRegF()
4924 %{
4925 constraint(ALLOC_IN_RC(float_reg));
4926 match(RegF);
4927
4928 op_cost(0);
4929 format %{ %}
4930 interface(REG_INTER);
4931 %}
4932
4933 // Double Register
4934 // Double register operands
4935 operand vRegD()
4936 %{
4937 constraint(ALLOC_IN_RC(double_reg));
4938 match(RegD);
4939
4940 op_cost(0);
4941 format %{ %}
4942 interface(REG_INTER);
4943 %}
4944
4945 // Generic vector class. This will be used for
4946 // all vector operands, including NEON and SVE.
4947 operand vReg()
4948 %{
4949 constraint(ALLOC_IN_RC(dynamic));
4950 match(VecA);
4951 match(VecD);
4952 match(VecX);
4953
4954 op_cost(0);
4955 format %{ %}
4956 interface(REG_INTER);
4957 %}
4958
4959 operand vReg_V10()
4960 %{
4961 constraint(ALLOC_IN_RC(v10_veca_reg));
4962 match(vReg);
4963
4964 op_cost(0);
4965 format %{ %}
4966 interface(REG_INTER);
4967 %}
4968
4969 operand vReg_V11()
4970 %{
4971 constraint(ALLOC_IN_RC(v11_veca_reg));
4972 match(vReg);
4973
4974 op_cost(0);
4975 format %{ %}
4976 interface(REG_INTER);
4977 %}
4978
4979 operand vReg_V12()
4980 %{
4981 constraint(ALLOC_IN_RC(v12_veca_reg));
4982 match(vReg);
4983
4984 op_cost(0);
4985 format %{ %}
4986 interface(REG_INTER);
4987 %}
4988
4989 operand vReg_V13()
4990 %{
4991 constraint(ALLOC_IN_RC(v13_veca_reg));
4992 match(vReg);
4993
4994 op_cost(0);
4995 format %{ %}
4996 interface(REG_INTER);
4997 %}
4998
4999 operand vReg_V17()
5000 %{
5001 constraint(ALLOC_IN_RC(v17_veca_reg));
5002 match(vReg);
5003
5004 op_cost(0);
5005 format %{ %}
5006 interface(REG_INTER);
5007 %}
5008
5009 operand vReg_V18()
5010 %{
5011 constraint(ALLOC_IN_RC(v18_veca_reg));
5012 match(vReg);
5013
5014 op_cost(0);
5015 format %{ %}
5016 interface(REG_INTER);
5017 %}
5018
5019 operand vReg_V23()
5020 %{
5021 constraint(ALLOC_IN_RC(v23_veca_reg));
5022 match(vReg);
5023
5024 op_cost(0);
5025 format %{ %}
5026 interface(REG_INTER);
5027 %}
5028
5029 operand vReg_V24()
5030 %{
5031 constraint(ALLOC_IN_RC(v24_veca_reg));
5032 match(vReg);
5033
5034 op_cost(0);
5035 format %{ %}
5036 interface(REG_INTER);
5037 %}
5038
5039 operand vecA()
5040 %{
5041 constraint(ALLOC_IN_RC(vectora_reg));
5042 match(VecA);
5043
5044 op_cost(0);
5045 format %{ %}
5046 interface(REG_INTER);
5047 %}
5048
5049 operand vecD()
5050 %{
5051 constraint(ALLOC_IN_RC(vectord_reg));
5052 match(VecD);
5053
5054 op_cost(0);
5055 format %{ %}
5056 interface(REG_INTER);
5057 %}
5058
5059 operand vecX()
5060 %{
5061 constraint(ALLOC_IN_RC(vectorx_reg));
5062 match(VecX);
5063
5064 op_cost(0);
5065 format %{ %}
5066 interface(REG_INTER);
5067 %}
5068
5069 operand vRegD_V0()
5070 %{
5071 constraint(ALLOC_IN_RC(v0_reg));
5072 match(RegD);
5073 op_cost(0);
5074 format %{ %}
5075 interface(REG_INTER);
5076 %}
5077
5078 operand vRegD_V1()
5079 %{
5080 constraint(ALLOC_IN_RC(v1_reg));
5081 match(RegD);
5082 op_cost(0);
5083 format %{ %}
5084 interface(REG_INTER);
5085 %}
5086
5087 operand vRegD_V2()
5088 %{
5089 constraint(ALLOC_IN_RC(v2_reg));
5090 match(RegD);
5091 op_cost(0);
5092 format %{ %}
5093 interface(REG_INTER);
5094 %}
5095
5096 operand vRegD_V3()
5097 %{
5098 constraint(ALLOC_IN_RC(v3_reg));
5099 match(RegD);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105 operand vRegD_V4()
5106 %{
5107 constraint(ALLOC_IN_RC(v4_reg));
5108 match(RegD);
5109 op_cost(0);
5110 format %{ %}
5111 interface(REG_INTER);
5112 %}
5113
5114 operand vRegD_V5()
5115 %{
5116 constraint(ALLOC_IN_RC(v5_reg));
5117 match(RegD);
5118 op_cost(0);
5119 format %{ %}
5120 interface(REG_INTER);
5121 %}
5122
5123 operand vRegD_V6()
5124 %{
5125 constraint(ALLOC_IN_RC(v6_reg));
5126 match(RegD);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 operand vRegD_V7()
5133 %{
5134 constraint(ALLOC_IN_RC(v7_reg));
5135 match(RegD);
5136 op_cost(0);
5137 format %{ %}
5138 interface(REG_INTER);
5139 %}
5140
5141 operand vRegD_V12()
5142 %{
5143 constraint(ALLOC_IN_RC(v12_reg));
5144 match(RegD);
5145 op_cost(0);
5146 format %{ %}
5147 interface(REG_INTER);
5148 %}
5149
5150 operand vRegD_V13()
5151 %{
5152 constraint(ALLOC_IN_RC(v13_reg));
5153 match(RegD);
5154 op_cost(0);
5155 format %{ %}
5156 interface(REG_INTER);
5157 %}
5158
5159 operand pReg()
5160 %{
5161 constraint(ALLOC_IN_RC(pr_reg));
5162 match(RegVectMask);
5163 match(pRegGov);
5164 op_cost(0);
5165 format %{ %}
5166 interface(REG_INTER);
5167 %}
5168
5169 operand pRegGov()
5170 %{
5171 constraint(ALLOC_IN_RC(gov_pr));
5172 match(RegVectMask);
5173 match(pReg);
5174 op_cost(0);
5175 format %{ %}
5176 interface(REG_INTER);
5177 %}
5178
5179 operand pRegGov_P0()
5180 %{
5181 constraint(ALLOC_IN_RC(p0_reg));
5182 match(RegVectMask);
5183 op_cost(0);
5184 format %{ %}
5185 interface(REG_INTER);
5186 %}
5187
5188 operand pRegGov_P1()
5189 %{
5190 constraint(ALLOC_IN_RC(p1_reg));
5191 match(RegVectMask);
5192 op_cost(0);
5193 format %{ %}
5194 interface(REG_INTER);
5195 %}
5196
5197 // Flags register, used as output of signed compare instructions
5198
5199 // note that on AArch64 we also use this register as the output for
5200 // for floating point compare instructions (CmpF CmpD). this ensures
5201 // that ordered inequality tests use GT, GE, LT or LE none of which
5202 // pass through cases where the result is unordered i.e. one or both
5203 // inputs to the compare is a NaN. this means that the ideal code can
5204 // replace e.g. a GT with an LE and not end up capturing the NaN case
5205 // (where the comparison should always fail). EQ and NE tests are
5206 // always generated in ideal code so that unordered folds into the NE
5207 // case, matching the behaviour of AArch64 NE.
5208 //
5209 // This differs from x86 where the outputs of FP compares use a
5210 // special FP flags registers and where compares based on this
5211 // register are distinguished into ordered inequalities (cmpOpUCF) and
5212 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5213 // to explicitly handle the unordered case in branches. x86 also has
5214 // to include extra CMoveX rules to accept a cmpOpUCF input.
5215
5216 operand rFlagsReg()
5217 %{
5218 constraint(ALLOC_IN_RC(int_flags));
5219 match(RegFlags);
5220
5221 op_cost(0);
5222 format %{ "RFLAGS" %}
5223 interface(REG_INTER);
5224 %}
5225
5226 // Flags register, used as output of unsigned compare instructions
5227 operand rFlagsRegU()
5228 %{
5229 constraint(ALLOC_IN_RC(int_flags));
5230 match(RegFlags);
5231
5232 op_cost(0);
5233 format %{ "RFLAGSU" %}
5234 interface(REG_INTER);
5235 %}
5236
5237 // Special Registers
5238
5239 // Method Register
5240 operand inline_cache_RegP(iRegP reg)
5241 %{
5242 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5243 match(reg);
5244 match(iRegPNoSp);
5245 op_cost(0);
5246 format %{ %}
5247 interface(REG_INTER);
5248 %}
5249
5250 // Thread Register
5251 operand thread_RegP(iRegP reg)
5252 %{
5253 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5254 match(reg);
5255 op_cost(0);
5256 format %{ %}
5257 interface(REG_INTER);
5258 %}
5259
5260 //----------Memory Operands----------------------------------------------------
5261
5262 operand indirect(iRegP reg)
5263 %{
5264 constraint(ALLOC_IN_RC(ptr_reg));
5265 match(reg);
5266 op_cost(0);
5267 format %{ "[$reg]" %}
5268 interface(MEMORY_INTER) %{
5269 base($reg);
5270 index(0xffffffff);
5271 scale(0x0);
5272 disp(0x0);
5273 %}
5274 %}
5275
5276 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5277 %{
5278 constraint(ALLOC_IN_RC(ptr_reg));
5279 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5280 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5281 op_cost(0);
5282 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5283 interface(MEMORY_INTER) %{
5284 base($reg);
5285 index($ireg);
5286 scale($scale);
5287 disp(0x0);
5288 %}
5289 %}
5290
5291 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5292 %{
5293 constraint(ALLOC_IN_RC(ptr_reg));
5294 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5295 match(AddP reg (LShiftL lreg scale));
5296 op_cost(0);
5297 format %{ "$reg, $lreg lsl($scale)" %}
5298 interface(MEMORY_INTER) %{
5299 base($reg);
5300 index($lreg);
5301 scale($scale);
5302 disp(0x0);
5303 %}
5304 %}
5305
5306 operand indIndexI2L(iRegP reg, iRegI ireg)
5307 %{
5308 constraint(ALLOC_IN_RC(ptr_reg));
5309 match(AddP reg (ConvI2L ireg));
5310 op_cost(0);
5311 format %{ "$reg, $ireg, 0, I2L" %}
5312 interface(MEMORY_INTER) %{
5313 base($reg);
5314 index($ireg);
5315 scale(0x0);
5316 disp(0x0);
5317 %}
5318 %}
5319
5320 operand indIndex(iRegP reg, iRegL lreg)
5321 %{
5322 constraint(ALLOC_IN_RC(ptr_reg));
5323 match(AddP reg lreg);
5324 op_cost(0);
5325 format %{ "$reg, $lreg" %}
5326 interface(MEMORY_INTER) %{
5327 base($reg);
5328 index($lreg);
5329 scale(0x0);
5330 disp(0x0);
5331 %}
5332 %}
5333
5334 operand indOffI1(iRegP reg, immIOffset1 off)
5335 %{
5336 constraint(ALLOC_IN_RC(ptr_reg));
5337 match(AddP reg off);
5338 op_cost(0);
5339 format %{ "[$reg, $off]" %}
5340 interface(MEMORY_INTER) %{
5341 base($reg);
5342 index(0xffffffff);
5343 scale(0x0);
5344 disp($off);
5345 %}
5346 %}
5347
5348 operand indOffI2(iRegP reg, immIOffset2 off)
5349 %{
5350 constraint(ALLOC_IN_RC(ptr_reg));
5351 match(AddP reg off);
5352 op_cost(0);
5353 format %{ "[$reg, $off]" %}
5354 interface(MEMORY_INTER) %{
5355 base($reg);
5356 index(0xffffffff);
5357 scale(0x0);
5358 disp($off);
5359 %}
5360 %}
5361
5362 operand indOffI4(iRegP reg, immIOffset4 off)
5363 %{
5364 constraint(ALLOC_IN_RC(ptr_reg));
5365 match(AddP reg off);
5366 op_cost(0);
5367 format %{ "[$reg, $off]" %}
5368 interface(MEMORY_INTER) %{
5369 base($reg);
5370 index(0xffffffff);
5371 scale(0x0);
5372 disp($off);
5373 %}
5374 %}
5375
5376 operand indOffI8(iRegP reg, immIOffset8 off)
5377 %{
5378 constraint(ALLOC_IN_RC(ptr_reg));
5379 match(AddP reg off);
5380 op_cost(0);
5381 format %{ "[$reg, $off]" %}
5382 interface(MEMORY_INTER) %{
5383 base($reg);
5384 index(0xffffffff);
5385 scale(0x0);
5386 disp($off);
5387 %}
5388 %}
5389
5390 operand indOffI16(iRegP reg, immIOffset16 off)
5391 %{
5392 constraint(ALLOC_IN_RC(ptr_reg));
5393 match(AddP reg off);
5394 op_cost(0);
5395 format %{ "[$reg, $off]" %}
5396 interface(MEMORY_INTER) %{
5397 base($reg);
5398 index(0xffffffff);
5399 scale(0x0);
5400 disp($off);
5401 %}
5402 %}
5403
5404 operand indOffL1(iRegP reg, immLoffset1 off)
5405 %{
5406 constraint(ALLOC_IN_RC(ptr_reg));
5407 match(AddP reg off);
5408 op_cost(0);
5409 format %{ "[$reg, $off]" %}
5410 interface(MEMORY_INTER) %{
5411 base($reg);
5412 index(0xffffffff);
5413 scale(0x0);
5414 disp($off);
5415 %}
5416 %}
5417
5418 operand indOffL2(iRegP reg, immLoffset2 off)
5419 %{
5420 constraint(ALLOC_IN_RC(ptr_reg));
5421 match(AddP reg off);
5422 op_cost(0);
5423 format %{ "[$reg, $off]" %}
5424 interface(MEMORY_INTER) %{
5425 base($reg);
5426 index(0xffffffff);
5427 scale(0x0);
5428 disp($off);
5429 %}
5430 %}
5431
5432 operand indOffL4(iRegP reg, immLoffset4 off)
5433 %{
5434 constraint(ALLOC_IN_RC(ptr_reg));
5435 match(AddP reg off);
5436 op_cost(0);
5437 format %{ "[$reg, $off]" %}
5438 interface(MEMORY_INTER) %{
5439 base($reg);
5440 index(0xffffffff);
5441 scale(0x0);
5442 disp($off);
5443 %}
5444 %}
5445
5446 operand indOffL8(iRegP reg, immLoffset8 off)
5447 %{
5448 constraint(ALLOC_IN_RC(ptr_reg));
5449 match(AddP reg off);
5450 op_cost(0);
5451 format %{ "[$reg, $off]" %}
5452 interface(MEMORY_INTER) %{
5453 base($reg);
5454 index(0xffffffff);
5455 scale(0x0);
5456 disp($off);
5457 %}
5458 %}
5459
5460 operand indOffL16(iRegP reg, immLoffset16 off)
5461 %{
5462 constraint(ALLOC_IN_RC(ptr_reg));
5463 match(AddP reg off);
5464 op_cost(0);
5465 format %{ "[$reg, $off]" %}
5466 interface(MEMORY_INTER) %{
5467 base($reg);
5468 index(0xffffffff);
5469 scale(0x0);
5470 disp($off);
5471 %}
5472 %}
5473
5474 operand indirectX2P(iRegL reg)
5475 %{
5476 constraint(ALLOC_IN_RC(ptr_reg));
5477 match(CastX2P reg);
5478 op_cost(0);
5479 format %{ "[$reg]\t# long -> ptr" %}
5480 interface(MEMORY_INTER) %{
5481 base($reg);
5482 index(0xffffffff);
5483 scale(0x0);
5484 disp(0x0);
5485 %}
5486 %}
5487
5488 operand indOffX2P(iRegL reg, immLOffset off)
5489 %{
5490 constraint(ALLOC_IN_RC(ptr_reg));
5491 match(AddP (CastX2P reg) off);
5492 op_cost(0);
5493 format %{ "[$reg, $off]\t# long -> ptr" %}
5494 interface(MEMORY_INTER) %{
5495 base($reg);
5496 index(0xffffffff);
5497 scale(0x0);
5498 disp($off);
5499 %}
5500 %}
5501
5502 operand indirectN(iRegN reg)
5503 %{
5504 predicate(CompressedOops::shift() == 0);
5505 constraint(ALLOC_IN_RC(ptr_reg));
5506 match(DecodeN reg);
5507 op_cost(0);
5508 format %{ "[$reg]\t# narrow" %}
5509 interface(MEMORY_INTER) %{
5510 base($reg);
5511 index(0xffffffff);
5512 scale(0x0);
5513 disp(0x0);
5514 %}
5515 %}
5516
5517 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5518 %{
5519 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5520 constraint(ALLOC_IN_RC(ptr_reg));
5521 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5522 op_cost(0);
5523 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5524 interface(MEMORY_INTER) %{
5525 base($reg);
5526 index($ireg);
5527 scale($scale);
5528 disp(0x0);
5529 %}
5530 %}
5531
5532 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5533 %{
5534 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5535 constraint(ALLOC_IN_RC(ptr_reg));
5536 match(AddP (DecodeN reg) (LShiftL lreg scale));
5537 op_cost(0);
5538 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5539 interface(MEMORY_INTER) %{
5540 base($reg);
5541 index($lreg);
5542 scale($scale);
5543 disp(0x0);
5544 %}
5545 %}
5546
5547 operand indIndexI2LN(iRegN reg, iRegI ireg)
5548 %{
5549 predicate(CompressedOops::shift() == 0);
5550 constraint(ALLOC_IN_RC(ptr_reg));
5551 match(AddP (DecodeN reg) (ConvI2L ireg));
5552 op_cost(0);
5553 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5554 interface(MEMORY_INTER) %{
5555 base($reg);
5556 index($ireg);
5557 scale(0x0);
5558 disp(0x0);
5559 %}
5560 %}
5561
5562 operand indIndexN(iRegN reg, iRegL lreg)
5563 %{
5564 predicate(CompressedOops::shift() == 0);
5565 constraint(ALLOC_IN_RC(ptr_reg));
5566 match(AddP (DecodeN reg) lreg);
5567 op_cost(0);
5568 format %{ "$reg, $lreg\t# narrow" %}
5569 interface(MEMORY_INTER) %{
5570 base($reg);
5571 index($lreg);
5572 scale(0x0);
5573 disp(0x0);
5574 %}
5575 %}
5576
5577 operand indOffIN(iRegN reg, immIOffset off)
5578 %{
5579 predicate(CompressedOops::shift() == 0);
5580 constraint(ALLOC_IN_RC(ptr_reg));
5581 match(AddP (DecodeN reg) off);
5582 op_cost(0);
5583 format %{ "[$reg, $off]\t# narrow" %}
5584 interface(MEMORY_INTER) %{
5585 base($reg);
5586 index(0xffffffff);
5587 scale(0x0);
5588 disp($off);
5589 %}
5590 %}
5591
5592 operand indOffLN(iRegN reg, immLOffset off)
5593 %{
5594 predicate(CompressedOops::shift() == 0);
5595 constraint(ALLOC_IN_RC(ptr_reg));
5596 match(AddP (DecodeN reg) off);
5597 op_cost(0);
5598 format %{ "[$reg, $off]\t# narrow" %}
5599 interface(MEMORY_INTER) %{
5600 base($reg);
5601 index(0xffffffff);
5602 scale(0x0);
5603 disp($off);
5604 %}
5605 %}
5606
5607
5608 //----------Special Memory Operands--------------------------------------------
5609 // Stack Slot Operand - This operand is used for loading and storing temporary
5610 // values on the stack where a match requires a value to
5611 // flow through memory.
5612 operand stackSlotP(sRegP reg)
5613 %{
5614 constraint(ALLOC_IN_RC(stack_slots));
5615 op_cost(100);
5616 // No match rule because this operand is only generated in matching
5617 // match(RegP);
5618 format %{ "[$reg]" %}
5619 interface(MEMORY_INTER) %{
5620 base(0x1e); // RSP
5621 index(0x0); // No Index
5622 scale(0x0); // No Scale
5623 disp($reg); // Stack Offset
5624 %}
5625 %}
5626
5627 operand stackSlotI(sRegI reg)
5628 %{
5629 constraint(ALLOC_IN_RC(stack_slots));
5630 // No match rule because this operand is only generated in matching
5631 // match(RegI);
5632 format %{ "[$reg]" %}
5633 interface(MEMORY_INTER) %{
5634 base(0x1e); // RSP
5635 index(0x0); // No Index
5636 scale(0x0); // No Scale
5637 disp($reg); // Stack Offset
5638 %}
5639 %}
5640
5641 operand stackSlotF(sRegF reg)
5642 %{
5643 constraint(ALLOC_IN_RC(stack_slots));
5644 // No match rule because this operand is only generated in matching
5645 // match(RegF);
5646 format %{ "[$reg]" %}
5647 interface(MEMORY_INTER) %{
5648 base(0x1e); // RSP
5649 index(0x0); // No Index
5650 scale(0x0); // No Scale
5651 disp($reg); // Stack Offset
5652 %}
5653 %}
5654
5655 operand stackSlotD(sRegD reg)
5656 %{
5657 constraint(ALLOC_IN_RC(stack_slots));
5658 // No match rule because this operand is only generated in matching
5659 // match(RegD);
5660 format %{ "[$reg]" %}
5661 interface(MEMORY_INTER) %{
5662 base(0x1e); // RSP
5663 index(0x0); // No Index
5664 scale(0x0); // No Scale
5665 disp($reg); // Stack Offset
5666 %}
5667 %}
5668
5669 operand stackSlotL(sRegL reg)
5670 %{
5671 constraint(ALLOC_IN_RC(stack_slots));
5672 // No match rule because this operand is only generated in matching
5673 // match(RegL);
5674 format %{ "[$reg]" %}
5675 interface(MEMORY_INTER) %{
5676 base(0x1e); // RSP
5677 index(0x0); // No Index
5678 scale(0x0); // No Scale
5679 disp($reg); // Stack Offset
5680 %}
5681 %}
5682
5683 // Operands for expressing Control Flow
5684 // NOTE: Label is a predefined operand which should not be redefined in
5685 // the AD file. It is generically handled within the ADLC.
5686
5687 //----------Conditional Branch Operands----------------------------------------
5688 // Comparison Op - This is the operation of the comparison, and is limited to
5689 // the following set of codes:
5690 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5691 //
5692 // Other attributes of the comparison, such as unsignedness, are specified
5693 // by the comparison instruction that sets a condition code flags register.
5694 // That result is represented by a flags operand whose subtype is appropriate
5695 // to the unsignedness (etc.) of the comparison.
5696 //
5697 // Later, the instruction which matches both the Comparison Op (a Bool) and
5698 // the flags (produced by the Cmp) specifies the coding of the comparison op
5699 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5700
5701 // used for signed integral comparisons and fp comparisons
5702
5703 operand cmpOp()
5704 %{
5705 match(Bool);
5706
5707 format %{ "" %}
5708 interface(COND_INTER) %{
5709 equal(0x0, "eq");
5710 not_equal(0x1, "ne");
5711 less(0xb, "lt");
5712 greater_equal(0xa, "ge");
5713 less_equal(0xd, "le");
5714 greater(0xc, "gt");
5715 overflow(0x6, "vs");
5716 no_overflow(0x7, "vc");
5717 %}
5718 %}
5719
5720 // used for unsigned integral comparisons
5721
5722 operand cmpOpU()
5723 %{
5724 match(Bool);
5725
5726 format %{ "" %}
5727 interface(COND_INTER) %{
5728 equal(0x0, "eq");
5729 not_equal(0x1, "ne");
5730 less(0x3, "lo");
5731 greater_equal(0x2, "hs");
5732 less_equal(0x9, "ls");
5733 greater(0x8, "hi");
5734 overflow(0x6, "vs");
5735 no_overflow(0x7, "vc");
5736 %}
5737 %}
5738
5739 // used for certain integral comparisons which can be
5740 // converted to cbxx or tbxx instructions
5741
5742 operand cmpOpEqNe()
5743 %{
5744 match(Bool);
5745 op_cost(0);
5746 predicate(n->as_Bool()->_test._test == BoolTest::ne
5747 || n->as_Bool()->_test._test == BoolTest::eq);
5748
5749 format %{ "" %}
5750 interface(COND_INTER) %{
5751 equal(0x0, "eq");
5752 not_equal(0x1, "ne");
5753 less(0xb, "lt");
5754 greater_equal(0xa, "ge");
5755 less_equal(0xd, "le");
5756 greater(0xc, "gt");
5757 overflow(0x6, "vs");
5758 no_overflow(0x7, "vc");
5759 %}
5760 %}
5761
5762 // used for certain integral comparisons which can be
5763 // converted to cbxx or tbxx instructions
5764
5765 operand cmpOpLtGe()
5766 %{
5767 match(Bool);
5768 op_cost(0);
5769
5770 predicate(n->as_Bool()->_test._test == BoolTest::lt
5771 || n->as_Bool()->_test._test == BoolTest::ge);
5772
5773 format %{ "" %}
5774 interface(COND_INTER) %{
5775 equal(0x0, "eq");
5776 not_equal(0x1, "ne");
5777 less(0xb, "lt");
5778 greater_equal(0xa, "ge");
5779 less_equal(0xd, "le");
5780 greater(0xc, "gt");
5781 overflow(0x6, "vs");
5782 no_overflow(0x7, "vc");
5783 %}
5784 %}
5785
5786 // used for certain unsigned integral comparisons which can be
5787 // converted to cbxx or tbxx instructions
5788
5789 operand cmpOpUEqNeLeGt()
5790 %{
5791 match(Bool);
5792 op_cost(0);
5793
5794 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5795 n->as_Bool()->_test._test == BoolTest::ne ||
5796 n->as_Bool()->_test._test == BoolTest::le ||
5797 n->as_Bool()->_test._test == BoolTest::gt);
5798
5799 format %{ "" %}
5800 interface(COND_INTER) %{
5801 equal(0x0, "eq");
5802 not_equal(0x1, "ne");
5803 less(0x3, "lo");
5804 greater_equal(0x2, "hs");
5805 less_equal(0x9, "ls");
5806 greater(0x8, "hi");
5807 overflow(0x6, "vs");
5808 no_overflow(0x7, "vc");
5809 %}
5810 %}
5811
5812 // Special operand allowing long args to int ops to be truncated for free
5813
5814 operand iRegL2I(iRegL reg) %{
5815
5816 op_cost(0);
5817
5818 match(ConvL2I reg);
5819
5820 format %{ "l2i($reg)" %}
5821
5822 interface(REG_INTER)
5823 %}
5824
5825 operand iRegL2P(iRegL reg) %{
5826
5827 op_cost(0);
5828
5829 match(CastX2P reg);
5830
5831 format %{ "l2p($reg)" %}
5832
5833 interface(REG_INTER)
5834 %}
5835
5836 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5837 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5838 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5839 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5840
5841 //----------OPERAND CLASSES----------------------------------------------------
5842 // Operand Classes are groups of operands that are used as to simplify
5843 // instruction definitions by not requiring the AD writer to specify
5844 // separate instructions for every form of operand when the
5845 // instruction accepts multiple operand types with the same basic
5846 // encoding and format. The classic case of this is memory operands.
5847
5848 // memory is used to define read/write location for load/store
5849 // instruction defs. we can turn a memory op into an Address
5850
5851 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5852 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5853
5854 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5855 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5856
5857 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5858 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5859
5860 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5861 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5862
5863 // All of the memory operands. For the pipeline description.
5864 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5865 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5866 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5867
5868
5869 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5870 // operations. it allows the src to be either an iRegI or a (ConvL2I
5871 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5872 // can be elided because the 32-bit instruction will just employ the
5873 // lower 32 bits anyway.
5874 //
5875 // n.b. this does not elide all L2I conversions. if the truncated
5876 // value is consumed by more than one operation then the ConvL2I
5877 // cannot be bundled into the consuming nodes so an l2i gets planted
5878 // (actually a movw $dst $src) and the downstream instructions consume
5879 // the result of the l2i as an iRegI input. That's a shame since the
5880 // movw is actually redundant but its not too costly.
5881
5882 opclass iRegIorL2I(iRegI, iRegL2I);
5883 opclass iRegPorL2P(iRegP, iRegL2P);
5884
5885 //----------PIPELINE-----------------------------------------------------------
5886 // Rules which define the behavior of the target architectures pipeline.
5887
5888 // For specific pipelines, eg A53, define the stages of that pipeline
5889 //pipe_desc(ISS, EX1, EX2, WR);
5890 #define ISS S0
5891 #define EX1 S1
5892 #define EX2 S2
5893 #define WR S3
5894
5895 // Integer ALU reg operation
5896 pipeline %{
5897
5898 attributes %{
5899 // ARM instructions are of fixed length
5900 fixed_size_instructions; // Fixed size instructions TODO does
5901 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5902 // ARM instructions come in 32-bit word units
5903 instruction_unit_size = 4; // An instruction is 4 bytes long
5904 instruction_fetch_unit_size = 64; // The processor fetches one line
5905 instruction_fetch_units = 1; // of 64 bytes
5906 %}
5907
5908 // We don't use an actual pipeline model so don't care about resources
5909 // or description. we do use pipeline classes to introduce fixed
5910 // latencies
5911
5912 //----------RESOURCES----------------------------------------------------------
5913 // Resources are the functional units available to the machine
5914
5915 resources( INS0, INS1, INS01 = INS0 | INS1,
5916 ALU0, ALU1, ALU = ALU0 | ALU1,
5917 MAC,
5918 DIV,
5919 BRANCH,
5920 LDST,
5921 NEON_FP);
5922
5923 //----------PIPELINE DESCRIPTION-----------------------------------------------
5924 // Pipeline Description specifies the stages in the machine's pipeline
5925
5926 // Define the pipeline as a generic 6 stage pipeline
5927 pipe_desc(S0, S1, S2, S3, S4, S5);
5928
5929 //----------PIPELINE CLASSES---------------------------------------------------
5930 // Pipeline Classes describe the stages in which input and output are
5931 // referenced by the hardware pipeline.
5932
5933 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5934 %{
5935 single_instruction;
5936 src1 : S1(read);
5937 src2 : S2(read);
5938 dst : S5(write);
5939 INS01 : ISS;
5940 NEON_FP : S5;
5941 %}
5942
5943 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5944 %{
5945 single_instruction;
5946 src1 : S1(read);
5947 src2 : S2(read);
5948 dst : S5(write);
5949 INS01 : ISS;
5950 NEON_FP : S5;
5951 %}
5952
5953 pipe_class fp_uop_s(vRegF dst, vRegF src)
5954 %{
5955 single_instruction;
5956 src : S1(read);
5957 dst : S5(write);
5958 INS01 : ISS;
5959 NEON_FP : S5;
5960 %}
5961
5962 pipe_class fp_uop_d(vRegD dst, vRegD src)
5963 %{
5964 single_instruction;
5965 src : S1(read);
5966 dst : S5(write);
5967 INS01 : ISS;
5968 NEON_FP : S5;
5969 %}
5970
5971 pipe_class fp_d2f(vRegF dst, vRegD src)
5972 %{
5973 single_instruction;
5974 src : S1(read);
5975 dst : S5(write);
5976 INS01 : ISS;
5977 NEON_FP : S5;
5978 %}
5979
5980 pipe_class fp_f2d(vRegD dst, vRegF src)
5981 %{
5982 single_instruction;
5983 src : S1(read);
5984 dst : S5(write);
5985 INS01 : ISS;
5986 NEON_FP : S5;
5987 %}
5988
5989 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5990 %{
5991 single_instruction;
5992 src : S1(read);
5993 dst : S5(write);
5994 INS01 : ISS;
5995 NEON_FP : S5;
5996 %}
5997
5998 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5999 %{
6000 single_instruction;
6001 src : S1(read);
6002 dst : S5(write);
6003 INS01 : ISS;
6004 NEON_FP : S5;
6005 %}
6006
6007 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6008 %{
6009 single_instruction;
6010 src : S1(read);
6011 dst : S5(write);
6012 INS01 : ISS;
6013 NEON_FP : S5;
6014 %}
6015
6016 pipe_class fp_l2f(vRegF dst, iRegL src)
6017 %{
6018 single_instruction;
6019 src : S1(read);
6020 dst : S5(write);
6021 INS01 : ISS;
6022 NEON_FP : S5;
6023 %}
6024
6025 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6026 %{
6027 single_instruction;
6028 src : S1(read);
6029 dst : S5(write);
6030 INS01 : ISS;
6031 NEON_FP : S5;
6032 %}
6033
6034 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6035 %{
6036 single_instruction;
6037 src : S1(read);
6038 dst : S5(write);
6039 INS01 : ISS;
6040 NEON_FP : S5;
6041 %}
6042
6043 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6044 %{
6045 single_instruction;
6046 src : S1(read);
6047 dst : S5(write);
6048 INS01 : ISS;
6049 NEON_FP : S5;
6050 %}
6051
6052 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6053 %{
6054 single_instruction;
6055 src : S1(read);
6056 dst : S5(write);
6057 INS01 : ISS;
6058 NEON_FP : S5;
6059 %}
6060
6061 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6062 %{
6063 single_instruction;
6064 src1 : S1(read);
6065 src2 : S2(read);
6066 dst : S5(write);
6067 INS0 : ISS;
6068 NEON_FP : S5;
6069 %}
6070
6071 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6072 %{
6073 single_instruction;
6074 src1 : S1(read);
6075 src2 : S2(read);
6076 dst : S5(write);
6077 INS0 : ISS;
6078 NEON_FP : S5;
6079 %}
6080
6081 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6082 %{
6083 single_instruction;
6084 cr : S1(read);
6085 src1 : S1(read);
6086 src2 : S1(read);
6087 dst : S3(write);
6088 INS01 : ISS;
6089 NEON_FP : S3;
6090 %}
6091
6092 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6093 %{
6094 single_instruction;
6095 cr : S1(read);
6096 src1 : S1(read);
6097 src2 : S1(read);
6098 dst : S3(write);
6099 INS01 : ISS;
6100 NEON_FP : S3;
6101 %}
6102
6103 pipe_class fp_imm_s(vRegF dst)
6104 %{
6105 single_instruction;
6106 dst : S3(write);
6107 INS01 : ISS;
6108 NEON_FP : S3;
6109 %}
6110
6111 pipe_class fp_imm_d(vRegD dst)
6112 %{
6113 single_instruction;
6114 dst : S3(write);
6115 INS01 : ISS;
6116 NEON_FP : S3;
6117 %}
6118
6119 pipe_class fp_load_constant_s(vRegF dst)
6120 %{
6121 single_instruction;
6122 dst : S4(write);
6123 INS01 : ISS;
6124 NEON_FP : S4;
6125 %}
6126
6127 pipe_class fp_load_constant_d(vRegD dst)
6128 %{
6129 single_instruction;
6130 dst : S4(write);
6131 INS01 : ISS;
6132 NEON_FP : S4;
6133 %}
6134
6135 //------- Integer ALU operations --------------------------
6136
6137 // Integer ALU reg-reg operation
6138 // Operands needed in EX1, result generated in EX2
6139 // Eg. ADD x0, x1, x2
6140 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6141 %{
6142 single_instruction;
6143 dst : EX2(write);
6144 src1 : EX1(read);
6145 src2 : EX1(read);
6146 INS01 : ISS; // Dual issue as instruction 0 or 1
6147 ALU : EX2;
6148 %}
6149
6150 // Integer ALU reg-reg operation with constant shift
6151 // Shifted register must be available in LATE_ISS instead of EX1
6152 // Eg. ADD x0, x1, x2, LSL #2
6153 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6154 %{
6155 single_instruction;
6156 dst : EX2(write);
6157 src1 : EX1(read);
6158 src2 : ISS(read);
6159 INS01 : ISS;
6160 ALU : EX2;
6161 %}
6162
6163 // Integer ALU reg operation with constant shift
6164 // Eg. LSL x0, x1, #shift
6165 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6166 %{
6167 single_instruction;
6168 dst : EX2(write);
6169 src1 : ISS(read);
6170 INS01 : ISS;
6171 ALU : EX2;
6172 %}
6173
6174 // Integer ALU reg-reg operation with variable shift
6175 // Both operands must be available in LATE_ISS instead of EX1
6176 // Result is available in EX1 instead of EX2
6177 // Eg. LSLV x0, x1, x2
6178 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6179 %{
6180 single_instruction;
6181 dst : EX1(write);
6182 src1 : ISS(read);
6183 src2 : ISS(read);
6184 INS01 : ISS;
6185 ALU : EX1;
6186 %}
6187
6188 // Integer ALU reg-reg operation with extract
6189 // As for _vshift above, but result generated in EX2
6190 // Eg. EXTR x0, x1, x2, #N
6191 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6192 %{
6193 single_instruction;
6194 dst : EX2(write);
6195 src1 : ISS(read);
6196 src2 : ISS(read);
6197 INS1 : ISS; // Can only dual issue as Instruction 1
6198 ALU : EX1;
6199 %}
6200
6201 // Integer ALU reg operation
6202 // Eg. NEG x0, x1
6203 pipe_class ialu_reg(iRegI dst, iRegI src)
6204 %{
6205 single_instruction;
6206 dst : EX2(write);
6207 src : EX1(read);
6208 INS01 : ISS;
6209 ALU : EX2;
6210 %}
6211
6212 // Integer ALU reg mmediate operation
6213 // Eg. ADD x0, x1, #N
6214 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6215 %{
6216 single_instruction;
6217 dst : EX2(write);
6218 src1 : EX1(read);
6219 INS01 : ISS;
6220 ALU : EX2;
6221 %}
6222
6223 // Integer ALU immediate operation (no source operands)
6224 // Eg. MOV x0, #N
6225 pipe_class ialu_imm(iRegI dst)
6226 %{
6227 single_instruction;
6228 dst : EX1(write);
6229 INS01 : ISS;
6230 ALU : EX1;
6231 %}
6232
6233 //------- Compare operation -------------------------------
6234
6235 // Compare reg-reg
6236 // Eg. CMP x0, x1
6237 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6238 %{
6239 single_instruction;
6240 // fixed_latency(16);
6241 cr : EX2(write);
6242 op1 : EX1(read);
6243 op2 : EX1(read);
6244 INS01 : ISS;
6245 ALU : EX2;
6246 %}
6247
6248 // Compare reg-reg
6249 // Eg. CMP x0, #N
6250 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6251 %{
6252 single_instruction;
6253 // fixed_latency(16);
6254 cr : EX2(write);
6255 op1 : EX1(read);
6256 INS01 : ISS;
6257 ALU : EX2;
6258 %}
6259
6260 //------- Conditional instructions ------------------------
6261
6262 // Conditional no operands
6263 // Eg. CSINC x0, zr, zr, <cond>
6264 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6265 %{
6266 single_instruction;
6267 cr : EX1(read);
6268 dst : EX2(write);
6269 INS01 : ISS;
6270 ALU : EX2;
6271 %}
6272
6273 // Conditional 2 operand
6274 // EG. CSEL X0, X1, X2, <cond>
6275 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6276 %{
6277 single_instruction;
6278 cr : EX1(read);
6279 src1 : EX1(read);
6280 src2 : EX1(read);
6281 dst : EX2(write);
6282 INS01 : ISS;
6283 ALU : EX2;
6284 %}
6285
6286 // Conditional 2 operand
6287 // EG. CSEL X0, X1, X2, <cond>
6288 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6289 %{
6290 single_instruction;
6291 cr : EX1(read);
6292 src : EX1(read);
6293 dst : EX2(write);
6294 INS01 : ISS;
6295 ALU : EX2;
6296 %}
6297
6298 //------- Multiply pipeline operations --------------------
6299
6300 // Multiply reg-reg
6301 // Eg. MUL w0, w1, w2
6302 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6303 %{
6304 single_instruction;
6305 dst : WR(write);
6306 src1 : ISS(read);
6307 src2 : ISS(read);
6308 INS01 : ISS;
6309 MAC : WR;
6310 %}
6311
6312 // Multiply accumulate
6313 // Eg. MADD w0, w1, w2, w3
6314 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6315 %{
6316 single_instruction;
6317 dst : WR(write);
6318 src1 : ISS(read);
6319 src2 : ISS(read);
6320 src3 : ISS(read);
6321 INS01 : ISS;
6322 MAC : WR;
6323 %}
6324
6325 // Eg. MUL w0, w1, w2
6326 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6327 %{
6328 single_instruction;
6329 fixed_latency(3); // Maximum latency for 64 bit mul
6330 dst : WR(write);
6331 src1 : ISS(read);
6332 src2 : ISS(read);
6333 INS01 : ISS;
6334 MAC : WR;
6335 %}
6336
6337 // Multiply accumulate
6338 // Eg. MADD w0, w1, w2, w3
6339 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6340 %{
6341 single_instruction;
6342 fixed_latency(3); // Maximum latency for 64 bit mul
6343 dst : WR(write);
6344 src1 : ISS(read);
6345 src2 : ISS(read);
6346 src3 : ISS(read);
6347 INS01 : ISS;
6348 MAC : WR;
6349 %}
6350
6351 //------- Divide pipeline operations --------------------
6352
6353 // Eg. SDIV w0, w1, w2
6354 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6355 %{
6356 single_instruction;
6357 fixed_latency(8); // Maximum latency for 32 bit divide
6358 dst : WR(write);
6359 src1 : ISS(read);
6360 src2 : ISS(read);
6361 INS0 : ISS; // Can only dual issue as instruction 0
6362 DIV : WR;
6363 %}
6364
6365 // Eg. SDIV x0, x1, x2
6366 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6367 %{
6368 single_instruction;
6369 fixed_latency(16); // Maximum latency for 64 bit divide
6370 dst : WR(write);
6371 src1 : ISS(read);
6372 src2 : ISS(read);
6373 INS0 : ISS; // Can only dual issue as instruction 0
6374 DIV : WR;
6375 %}
6376
6377 //------- Load pipeline operations ------------------------
6378
6379 // Load - prefetch
6380 // Eg. PFRM <mem>
6381 pipe_class iload_prefetch(memory mem)
6382 %{
6383 single_instruction;
6384 mem : ISS(read);
6385 INS01 : ISS;
6386 LDST : WR;
6387 %}
6388
6389 // Load - reg, mem
6390 // Eg. LDR x0, <mem>
6391 pipe_class iload_reg_mem(iRegI dst, memory mem)
6392 %{
6393 single_instruction;
6394 dst : WR(write);
6395 mem : ISS(read);
6396 INS01 : ISS;
6397 LDST : WR;
6398 %}
6399
6400 // Load - reg, reg
6401 // Eg. LDR x0, [sp, x1]
6402 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6403 %{
6404 single_instruction;
6405 dst : WR(write);
6406 src : ISS(read);
6407 INS01 : ISS;
6408 LDST : WR;
6409 %}
6410
6411 //------- Store pipeline operations -----------------------
6412
6413 // Store - zr, mem
6414 // Eg. STR zr, <mem>
6415 pipe_class istore_mem(memory mem)
6416 %{
6417 single_instruction;
6418 mem : ISS(read);
6419 INS01 : ISS;
6420 LDST : WR;
6421 %}
6422
6423 // Store - reg, mem
6424 // Eg. STR x0, <mem>
6425 pipe_class istore_reg_mem(iRegI src, memory mem)
6426 %{
6427 single_instruction;
6428 mem : ISS(read);
6429 src : EX2(read);
6430 INS01 : ISS;
6431 LDST : WR;
6432 %}
6433
6434 // Store - reg, reg
6435 // Eg. STR x0, [sp, x1]
6436 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6437 %{
6438 single_instruction;
6439 dst : ISS(read);
6440 src : EX2(read);
6441 INS01 : ISS;
6442 LDST : WR;
6443 %}
6444
6445 //------- Store pipeline operations -----------------------
6446
6447 // Branch
6448 pipe_class pipe_branch()
6449 %{
6450 single_instruction;
6451 INS01 : ISS;
6452 BRANCH : EX1;
6453 %}
6454
6455 // Conditional branch
6456 pipe_class pipe_branch_cond(rFlagsReg cr)
6457 %{
6458 single_instruction;
6459 cr : EX1(read);
6460 INS01 : ISS;
6461 BRANCH : EX1;
6462 %}
6463
6464 // Compare & Branch
6465 // EG. CBZ/CBNZ
6466 pipe_class pipe_cmp_branch(iRegI op1)
6467 %{
6468 single_instruction;
6469 op1 : EX1(read);
6470 INS01 : ISS;
6471 BRANCH : EX1;
6472 %}
6473
6474 //------- Synchronisation operations ----------------------
6475
6476 // Any operation requiring serialization.
6477 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6478 pipe_class pipe_serial()
6479 %{
6480 single_instruction;
6481 force_serialization;
6482 fixed_latency(16);
6483 INS01 : ISS(2); // Cannot dual issue with any other instruction
6484 LDST : WR;
6485 %}
6486
6487 // Generic big/slow expanded idiom - also serialized
6488 pipe_class pipe_slow()
6489 %{
6490 instruction_count(10);
6491 multiple_bundles;
6492 force_serialization;
6493 fixed_latency(16);
6494 INS01 : ISS(2); // Cannot dual issue with any other instruction
6495 LDST : WR;
6496 %}
6497
6498 // Empty pipeline class
6499 pipe_class pipe_class_empty()
6500 %{
6501 single_instruction;
6502 fixed_latency(0);
6503 %}
6504
6505 // Default pipeline class.
6506 pipe_class pipe_class_default()
6507 %{
6508 single_instruction;
6509 fixed_latency(2);
6510 %}
6511
6512 // Pipeline class for compares.
6513 pipe_class pipe_class_compare()
6514 %{
6515 single_instruction;
6516 fixed_latency(16);
6517 %}
6518
6519 // Pipeline class for memory operations.
6520 pipe_class pipe_class_memory()
6521 %{
6522 single_instruction;
6523 fixed_latency(16);
6524 %}
6525
6526 // Pipeline class for call.
6527 pipe_class pipe_class_call()
6528 %{
6529 single_instruction;
6530 fixed_latency(100);
6531 %}
6532
6533 // Define the class for the Nop node.
6534 define %{
6535 MachNop = pipe_class_empty;
6536 %}
6537
6538 %}
6539 //----------INSTRUCTIONS-------------------------------------------------------
6540 //
6541 // match -- States which machine-independent subtree may be replaced
6542 // by this instruction.
6543 // ins_cost -- The estimated cost of this instruction is used by instruction
6544 // selection to identify a minimum cost tree of machine
6545 // instructions that matches a tree of machine-independent
6546 // instructions.
6547 // format -- A string providing the disassembly for this instruction.
6548 // The value of an instruction's operand may be inserted
6549 // by referring to it with a '$' prefix.
6550 // opcode -- Three instruction opcodes may be provided. These are referred
6551 // to within an encode class as $primary, $secondary, and $tertiary
6552 // rrspectively. The primary opcode is commonly used to
6553 // indicate the type of machine instruction, while secondary
6554 // and tertiary are often used for prefix options or addressing
6555 // modes.
6556 // ins_encode -- A list of encode classes with parameters. The encode class
6557 // name must have been defined in an 'enc_class' specification
6558 // in the encode section of the architecture description.
6559
6560 // ============================================================================
6561 // Memory (Load/Store) Instructions
6562
6563 // Load Instructions
6564
6565 // Load Byte (8 bit signed)
6566 instruct loadB(iRegINoSp dst, memory1 mem)
6567 %{
6568 match(Set dst (LoadB mem));
6569 predicate(!needs_acquiring_load(n));
6570
6571 ins_cost(4 * INSN_COST);
6572 format %{ "ldrsbw $dst, $mem\t# byte" %}
6573
6574 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6575
6576 ins_pipe(iload_reg_mem);
6577 %}
6578
6579 // Load Byte (8 bit signed) into long
6580 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6581 %{
6582 match(Set dst (ConvI2L (LoadB mem)));
6583 predicate(!needs_acquiring_load(n->in(1)));
6584
6585 ins_cost(4 * INSN_COST);
6586 format %{ "ldrsb $dst, $mem\t# byte" %}
6587
6588 ins_encode(aarch64_enc_ldrsb(dst, mem));
6589
6590 ins_pipe(iload_reg_mem);
6591 %}
6592
6593 // Load Byte (8 bit unsigned)
6594 instruct loadUB(iRegINoSp dst, memory1 mem)
6595 %{
6596 match(Set dst (LoadUB mem));
6597 predicate(!needs_acquiring_load(n));
6598
6599 ins_cost(4 * INSN_COST);
6600 format %{ "ldrbw $dst, $mem\t# byte" %}
6601
6602 ins_encode(aarch64_enc_ldrb(dst, mem));
6603
6604 ins_pipe(iload_reg_mem);
6605 %}
6606
6607 // Load Byte (8 bit unsigned) into long
6608 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6609 %{
6610 match(Set dst (ConvI2L (LoadUB mem)));
6611 predicate(!needs_acquiring_load(n->in(1)));
6612
6613 ins_cost(4 * INSN_COST);
6614 format %{ "ldrb $dst, $mem\t# byte" %}
6615
6616 ins_encode(aarch64_enc_ldrb(dst, mem));
6617
6618 ins_pipe(iload_reg_mem);
6619 %}
6620
6621 // Load Short (16 bit signed)
6622 instruct loadS(iRegINoSp dst, memory2 mem)
6623 %{
6624 match(Set dst (LoadS mem));
6625 predicate(!needs_acquiring_load(n));
6626
6627 ins_cost(4 * INSN_COST);
6628 format %{ "ldrshw $dst, $mem\t# short" %}
6629
6630 ins_encode(aarch64_enc_ldrshw(dst, mem));
6631
6632 ins_pipe(iload_reg_mem);
6633 %}
6634
6635 // Load Short (16 bit signed) into long
6636 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6637 %{
6638 match(Set dst (ConvI2L (LoadS mem)));
6639 predicate(!needs_acquiring_load(n->in(1)));
6640
6641 ins_cost(4 * INSN_COST);
6642 format %{ "ldrsh $dst, $mem\t# short" %}
6643
6644 ins_encode(aarch64_enc_ldrsh(dst, mem));
6645
6646 ins_pipe(iload_reg_mem);
6647 %}
6648
6649 // Load Char (16 bit unsigned)
6650 instruct loadUS(iRegINoSp dst, memory2 mem)
6651 %{
6652 match(Set dst (LoadUS mem));
6653 predicate(!needs_acquiring_load(n));
6654
6655 ins_cost(4 * INSN_COST);
6656 format %{ "ldrh $dst, $mem\t# short" %}
6657
6658 ins_encode(aarch64_enc_ldrh(dst, mem));
6659
6660 ins_pipe(iload_reg_mem);
6661 %}
6662
6663 // Load Short/Char (16 bit unsigned) into long
6664 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6665 %{
6666 match(Set dst (ConvI2L (LoadUS mem)));
6667 predicate(!needs_acquiring_load(n->in(1)));
6668
6669 ins_cost(4 * INSN_COST);
6670 format %{ "ldrh $dst, $mem\t# short" %}
6671
6672 ins_encode(aarch64_enc_ldrh(dst, mem));
6673
6674 ins_pipe(iload_reg_mem);
6675 %}
6676
6677 // Load Integer (32 bit signed)
6678 instruct loadI(iRegINoSp dst, memory4 mem)
6679 %{
6680 match(Set dst (LoadI mem));
6681 predicate(!needs_acquiring_load(n));
6682
6683 ins_cost(4 * INSN_COST);
6684 format %{ "ldrw $dst, $mem\t# int" %}
6685
6686 ins_encode(aarch64_enc_ldrw(dst, mem));
6687
6688 ins_pipe(iload_reg_mem);
6689 %}
6690
6691 // Load Integer (32 bit signed) into long
6692 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6693 %{
6694 match(Set dst (ConvI2L (LoadI mem)));
6695 predicate(!needs_acquiring_load(n->in(1)));
6696
6697 ins_cost(4 * INSN_COST);
6698 format %{ "ldrsw $dst, $mem\t# int" %}
6699
6700 ins_encode(aarch64_enc_ldrsw(dst, mem));
6701
6702 ins_pipe(iload_reg_mem);
6703 %}
6704
6705 // Load Integer (32 bit unsigned) into long
6706 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6707 %{
6708 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6709 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6710
6711 ins_cost(4 * INSN_COST);
6712 format %{ "ldrw $dst, $mem\t# int" %}
6713
6714 ins_encode(aarch64_enc_ldrw(dst, mem));
6715
6716 ins_pipe(iload_reg_mem);
6717 %}
6718
6719 // Load Long (64 bit signed)
6720 instruct loadL(iRegLNoSp dst, memory8 mem)
6721 %{
6722 match(Set dst (LoadL mem));
6723 predicate(!needs_acquiring_load(n));
6724
6725 ins_cost(4 * INSN_COST);
6726 format %{ "ldr $dst, $mem\t# int" %}
6727
6728 ins_encode(aarch64_enc_ldr(dst, mem));
6729
6730 ins_pipe(iload_reg_mem);
6731 %}
6732
6733 // Load Range
6734 instruct loadRange(iRegINoSp dst, memory4 mem)
6735 %{
6736 match(Set dst (LoadRange mem));
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldrw $dst, $mem\t# range" %}
6740
6741 ins_encode(aarch64_enc_ldrw(dst, mem));
6742
6743 ins_pipe(iload_reg_mem);
6744 %}
6745
6746 // Load Pointer
6747 instruct loadP(iRegPNoSp dst, memory8 mem)
6748 %{
6749 match(Set dst (LoadP mem));
6750 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6751
6752 ins_cost(4 * INSN_COST);
6753 format %{ "ldr $dst, $mem\t# ptr" %}
6754
6755 ins_encode(aarch64_enc_ldr(dst, mem));
6756
6757 ins_pipe(iload_reg_mem);
6758 %}
6759
6760 // Load Compressed Pointer
6761 instruct loadN(iRegNNoSp dst, memory4 mem)
6762 %{
6763 match(Set dst (LoadN mem));
6764 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6765
6766 ins_cost(4 * INSN_COST);
6767 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6768
6769 ins_encode(aarch64_enc_ldrw(dst, mem));
6770
6771 ins_pipe(iload_reg_mem);
6772 %}
6773
6774 // Load Klass Pointer
6775 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6776 %{
6777 match(Set dst (LoadKlass mem));
6778 predicate(!needs_acquiring_load(n));
6779
6780 ins_cost(4 * INSN_COST);
6781 format %{ "ldr $dst, $mem\t# class" %}
6782
6783 ins_encode(aarch64_enc_ldr(dst, mem));
6784
6785 ins_pipe(iload_reg_mem);
6786 %}
6787
6788 // Load Narrow Klass Pointer
6789 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6790 %{
6791 match(Set dst (LoadNKlass mem));
6792 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6793
6794 ins_cost(4 * INSN_COST);
6795 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6796
6797 ins_encode(aarch64_enc_ldrw(dst, mem));
6798
6799 ins_pipe(iload_reg_mem);
6800 %}
6801
6802 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6803 %{
6804 match(Set dst (LoadNKlass mem));
6805 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6806
6807 ins_cost(4 * INSN_COST);
6808 format %{
6809 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6810 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6811 %}
6812 ins_encode %{
6813 // inlined aarch64_enc_ldrw
6814 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6815 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6816 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6817 %}
6818 ins_pipe(iload_reg_mem);
6819 %}
6820
6821 // Load Float
6822 instruct loadF(vRegF dst, memory4 mem)
6823 %{
6824 match(Set dst (LoadF mem));
6825 predicate(!needs_acquiring_load(n));
6826
6827 ins_cost(4 * INSN_COST);
6828 format %{ "ldrs $dst, $mem\t# float" %}
6829
6830 ins_encode( aarch64_enc_ldrs(dst, mem) );
6831
6832 ins_pipe(pipe_class_memory);
6833 %}
6834
6835 // Load Double
6836 instruct loadD(vRegD dst, memory8 mem)
6837 %{
6838 match(Set dst (LoadD mem));
6839 predicate(!needs_acquiring_load(n));
6840
6841 ins_cost(4 * INSN_COST);
6842 format %{ "ldrd $dst, $mem\t# double" %}
6843
6844 ins_encode( aarch64_enc_ldrd(dst, mem) );
6845
6846 ins_pipe(pipe_class_memory);
6847 %}
6848
6849
6850 // Load Int Constant
6851 instruct loadConI(iRegINoSp dst, immI src)
6852 %{
6853 match(Set dst src);
6854
6855 ins_cost(INSN_COST);
6856 format %{ "mov $dst, $src\t# int" %}
6857
6858 ins_encode( aarch64_enc_movw_imm(dst, src) );
6859
6860 ins_pipe(ialu_imm);
6861 %}
6862
6863 // Load Long Constant
6864 instruct loadConL(iRegLNoSp dst, immL src)
6865 %{
6866 match(Set dst src);
6867
6868 ins_cost(INSN_COST);
6869 format %{ "mov $dst, $src\t# long" %}
6870
6871 ins_encode( aarch64_enc_mov_imm(dst, src) );
6872
6873 ins_pipe(ialu_imm);
6874 %}
6875
6876 // Load Pointer Constant
6877
6878 instruct loadConP(iRegPNoSp dst, immP con)
6879 %{
6880 match(Set dst con);
6881
6882 ins_cost(INSN_COST * 4);
6883 format %{
6884 "mov $dst, $con\t# ptr\n\t"
6885 %}
6886
6887 ins_encode(aarch64_enc_mov_p(dst, con));
6888
6889 ins_pipe(ialu_imm);
6890 %}
6891
6892 // Load Null Pointer Constant
6893
6894 instruct loadConP0(iRegPNoSp dst, immP0 con)
6895 %{
6896 match(Set dst con);
6897
6898 ins_cost(INSN_COST);
6899 format %{ "mov $dst, $con\t# nullptr ptr" %}
6900
6901 ins_encode(aarch64_enc_mov_p0(dst, con));
6902
6903 ins_pipe(ialu_imm);
6904 %}
6905
6906 // Load Pointer Constant One
6907
6908 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6909 %{
6910 match(Set dst con);
6911
6912 ins_cost(INSN_COST);
6913 format %{ "mov $dst, $con\t# nullptr ptr" %}
6914
6915 ins_encode(aarch64_enc_mov_p1(dst, con));
6916
6917 ins_pipe(ialu_imm);
6918 %}
6919
6920 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6921 %{
6922 match(Set dst con);
6923
6924 ins_cost(INSN_COST);
6925 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6926
6927 ins_encode %{
6928 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6929 %}
6930
6931 ins_pipe(ialu_imm);
6932 %}
6933
6934 // Load Narrow Pointer Constant
6935
6936 instruct loadConN(iRegNNoSp dst, immN con)
6937 %{
6938 match(Set dst con);
6939
6940 ins_cost(INSN_COST * 4);
6941 format %{ "mov $dst, $con\t# compressed ptr" %}
6942
6943 ins_encode(aarch64_enc_mov_n(dst, con));
6944
6945 ins_pipe(ialu_imm);
6946 %}
6947
6948 // Load Narrow Null Pointer Constant
6949
6950 instruct loadConN0(iRegNNoSp dst, immN0 con)
6951 %{
6952 match(Set dst con);
6953
6954 ins_cost(INSN_COST);
6955 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6956
6957 ins_encode(aarch64_enc_mov_n0(dst, con));
6958
6959 ins_pipe(ialu_imm);
6960 %}
6961
6962 // Load Narrow Klass Constant
6963
6964 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6965 %{
6966 match(Set dst con);
6967
6968 ins_cost(INSN_COST);
6969 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6970
6971 ins_encode(aarch64_enc_mov_nk(dst, con));
6972
6973 ins_pipe(ialu_imm);
6974 %}
6975
6976 // Load Packed Float Constant
6977
6978 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6979 match(Set dst con);
6980 ins_cost(INSN_COST * 4);
6981 format %{ "fmovs $dst, $con"%}
6982 ins_encode %{
6983 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6984 %}
6985
6986 ins_pipe(fp_imm_s);
6987 %}
6988
6989 // Load Float Constant
6990
6991 instruct loadConF(vRegF dst, immF con) %{
6992 match(Set dst con);
6993
6994 ins_cost(INSN_COST * 4);
6995
6996 format %{
6997 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6998 %}
6999
7000 ins_encode %{
7001 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7002 %}
7003
7004 ins_pipe(fp_load_constant_s);
7005 %}
7006
7007 // Load Packed Double Constant
7008
7009 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7010 match(Set dst con);
7011 ins_cost(INSN_COST);
7012 format %{ "fmovd $dst, $con"%}
7013 ins_encode %{
7014 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7015 %}
7016
7017 ins_pipe(fp_imm_d);
7018 %}
7019
7020 // Load Double Constant
7021
7022 instruct loadConD(vRegD dst, immD con) %{
7023 match(Set dst con);
7024
7025 ins_cost(INSN_COST * 5);
7026 format %{
7027 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7028 %}
7029
7030 ins_encode %{
7031 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7032 %}
7033
7034 ins_pipe(fp_load_constant_d);
7035 %}
7036
7037 // Load Half Float Constant
7038 instruct loadConH(vRegF dst, immH con) %{
7039 match(Set dst con);
7040 format %{ "mov rscratch1, $con\n\t"
7041 "fmov $dst, rscratch1"
7042 %}
7043 ins_encode %{
7044 __ movw(rscratch1, (uint32_t)$con$$constant);
7045 __ fmovs($dst$$FloatRegister, rscratch1);
7046 %}
7047 ins_pipe(pipe_class_default);
7048 %}
7049
7050 // Store Instructions
7051
7052 // Store Byte
7053 instruct storeB(iRegIorL2I src, memory1 mem)
7054 %{
7055 match(Set mem (StoreB mem src));
7056 predicate(!needs_releasing_store(n));
7057
7058 ins_cost(INSN_COST);
7059 format %{ "strb $src, $mem\t# byte" %}
7060
7061 ins_encode(aarch64_enc_strb(src, mem));
7062
7063 ins_pipe(istore_reg_mem);
7064 %}
7065
7066
7067 instruct storeimmB0(immI0 zero, memory1 mem)
7068 %{
7069 match(Set mem (StoreB mem zero));
7070 predicate(!needs_releasing_store(n));
7071
7072 ins_cost(INSN_COST);
7073 format %{ "strb rscractch2, $mem\t# byte" %}
7074
7075 ins_encode(aarch64_enc_strb0(mem));
7076
7077 ins_pipe(istore_mem);
7078 %}
7079
7080 // Store Char/Short
7081 instruct storeC(iRegIorL2I src, memory2 mem)
7082 %{
7083 match(Set mem (StoreC mem src));
7084 predicate(!needs_releasing_store(n));
7085
7086 ins_cost(INSN_COST);
7087 format %{ "strh $src, $mem\t# short" %}
7088
7089 ins_encode(aarch64_enc_strh(src, mem));
7090
7091 ins_pipe(istore_reg_mem);
7092 %}
7093
7094 instruct storeimmC0(immI0 zero, memory2 mem)
7095 %{
7096 match(Set mem (StoreC mem zero));
7097 predicate(!needs_releasing_store(n));
7098
7099 ins_cost(INSN_COST);
7100 format %{ "strh zr, $mem\t# short" %}
7101
7102 ins_encode(aarch64_enc_strh0(mem));
7103
7104 ins_pipe(istore_mem);
7105 %}
7106
7107 // Store Integer
7108
7109 instruct storeI(iRegIorL2I src, memory4 mem)
7110 %{
7111 match(Set mem(StoreI mem src));
7112 predicate(!needs_releasing_store(n));
7113
7114 ins_cost(INSN_COST);
7115 format %{ "strw $src, $mem\t# int" %}
7116
7117 ins_encode(aarch64_enc_strw(src, mem));
7118
7119 ins_pipe(istore_reg_mem);
7120 %}
7121
7122 instruct storeimmI0(immI0 zero, memory4 mem)
7123 %{
7124 match(Set mem(StoreI mem zero));
7125 predicate(!needs_releasing_store(n));
7126
7127 ins_cost(INSN_COST);
7128 format %{ "strw zr, $mem\t# int" %}
7129
7130 ins_encode(aarch64_enc_strw0(mem));
7131
7132 ins_pipe(istore_mem);
7133 %}
7134
7135 // Store Long (64 bit signed)
7136 instruct storeL(iRegL src, memory8 mem)
7137 %{
7138 match(Set mem (StoreL mem src));
7139 predicate(!needs_releasing_store(n));
7140
7141 ins_cost(INSN_COST);
7142 format %{ "str $src, $mem\t# int" %}
7143
7144 ins_encode(aarch64_enc_str(src, mem));
7145
7146 ins_pipe(istore_reg_mem);
7147 %}
7148
7149 // Store Long (64 bit signed)
7150 instruct storeimmL0(immL0 zero, memory8 mem)
7151 %{
7152 match(Set mem (StoreL mem zero));
7153 predicate(!needs_releasing_store(n));
7154
7155 ins_cost(INSN_COST);
7156 format %{ "str zr, $mem\t# int" %}
7157
7158 ins_encode(aarch64_enc_str0(mem));
7159
7160 ins_pipe(istore_mem);
7161 %}
7162
7163 // Store Pointer
7164 instruct storeP(iRegP src, memory8 mem)
7165 %{
7166 match(Set mem (StoreP mem src));
7167 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7168
7169 ins_cost(INSN_COST);
7170 format %{ "str $src, $mem\t# ptr" %}
7171
7172 ins_encode(aarch64_enc_str(src, mem));
7173
7174 ins_pipe(istore_reg_mem);
7175 %}
7176
7177 // Store Pointer
7178 instruct storeimmP0(immP0 zero, memory8 mem)
7179 %{
7180 match(Set mem (StoreP mem zero));
7181 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7182
7183 ins_cost(INSN_COST);
7184 format %{ "str zr, $mem\t# ptr" %}
7185
7186 ins_encode(aarch64_enc_str0(mem));
7187
7188 ins_pipe(istore_mem);
7189 %}
7190
7191 // Store Compressed Pointer
7192 instruct storeN(iRegN src, memory4 mem)
7193 %{
7194 match(Set mem (StoreN mem src));
7195 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7196
7197 ins_cost(INSN_COST);
7198 format %{ "strw $src, $mem\t# compressed ptr" %}
7199
7200 ins_encode(aarch64_enc_strw(src, mem));
7201
7202 ins_pipe(istore_reg_mem);
7203 %}
7204
7205 instruct storeImmN0(immN0 zero, memory4 mem)
7206 %{
7207 match(Set mem (StoreN mem zero));
7208 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7209
7210 ins_cost(INSN_COST);
7211 format %{ "strw zr, $mem\t# compressed ptr" %}
7212
7213 ins_encode(aarch64_enc_strw0(mem));
7214
7215 ins_pipe(istore_mem);
7216 %}
7217
7218 // Store Float
7219 instruct storeF(vRegF src, memory4 mem)
7220 %{
7221 match(Set mem (StoreF mem src));
7222 predicate(!needs_releasing_store(n));
7223
7224 ins_cost(INSN_COST);
7225 format %{ "strs $src, $mem\t# float" %}
7226
7227 ins_encode( aarch64_enc_strs(src, mem) );
7228
7229 ins_pipe(pipe_class_memory);
7230 %}
7231
7232 // TODO
7233 // implement storeImmF0 and storeFImmPacked
7234
7235 // Store Double
7236 instruct storeD(vRegD src, memory8 mem)
7237 %{
7238 match(Set mem (StoreD mem src));
7239 predicate(!needs_releasing_store(n));
7240
7241 ins_cost(INSN_COST);
7242 format %{ "strd $src, $mem\t# double" %}
7243
7244 ins_encode( aarch64_enc_strd(src, mem) );
7245
7246 ins_pipe(pipe_class_memory);
7247 %}
7248
7249 // Store Compressed Klass Pointer
7250 instruct storeNKlass(iRegN src, memory4 mem)
7251 %{
7252 predicate(!needs_releasing_store(n));
7253 match(Set mem (StoreNKlass mem src));
7254
7255 ins_cost(INSN_COST);
7256 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7257
7258 ins_encode(aarch64_enc_strw(src, mem));
7259
7260 ins_pipe(istore_reg_mem);
7261 %}
7262
7263 // TODO
7264 // implement storeImmD0 and storeDImmPacked
7265
7266 // prefetch instructions
7267 // Must be safe to execute with invalid address (cannot fault).
7268
7269 instruct prefetchalloc( memory8 mem ) %{
7270 match(PrefetchAllocation mem);
7271
7272 ins_cost(INSN_COST);
7273 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7274
7275 ins_encode( aarch64_enc_prefetchw(mem) );
7276
7277 ins_pipe(iload_prefetch);
7278 %}
7279
7280 // ---------------- volatile loads and stores ----------------
7281
7282 // Load Byte (8 bit signed)
7283 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7284 %{
7285 match(Set dst (LoadB mem));
7286
7287 ins_cost(VOLATILE_REF_COST);
7288 format %{ "ldarsb $dst, $mem\t# byte" %}
7289
7290 ins_encode(aarch64_enc_ldarsb(dst, mem));
7291
7292 ins_pipe(pipe_serial);
7293 %}
7294
7295 // Load Byte (8 bit signed) into long
7296 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7297 %{
7298 match(Set dst (ConvI2L (LoadB mem)));
7299
7300 ins_cost(VOLATILE_REF_COST);
7301 format %{ "ldarsb $dst, $mem\t# byte" %}
7302
7303 ins_encode(aarch64_enc_ldarsb(dst, mem));
7304
7305 ins_pipe(pipe_serial);
7306 %}
7307
7308 // Load Byte (8 bit unsigned)
7309 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7310 %{
7311 match(Set dst (LoadUB mem));
7312
7313 ins_cost(VOLATILE_REF_COST);
7314 format %{ "ldarb $dst, $mem\t# byte" %}
7315
7316 ins_encode(aarch64_enc_ldarb(dst, mem));
7317
7318 ins_pipe(pipe_serial);
7319 %}
7320
7321 // Load Byte (8 bit unsigned) into long
7322 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7323 %{
7324 match(Set dst (ConvI2L (LoadUB mem)));
7325
7326 ins_cost(VOLATILE_REF_COST);
7327 format %{ "ldarb $dst, $mem\t# byte" %}
7328
7329 ins_encode(aarch64_enc_ldarb(dst, mem));
7330
7331 ins_pipe(pipe_serial);
7332 %}
7333
7334 // Load Short (16 bit signed)
7335 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7336 %{
7337 match(Set dst (LoadS mem));
7338
7339 ins_cost(VOLATILE_REF_COST);
7340 format %{ "ldarshw $dst, $mem\t# short" %}
7341
7342 ins_encode(aarch64_enc_ldarshw(dst, mem));
7343
7344 ins_pipe(pipe_serial);
7345 %}
7346
7347 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7348 %{
7349 match(Set dst (LoadUS mem));
7350
7351 ins_cost(VOLATILE_REF_COST);
7352 format %{ "ldarhw $dst, $mem\t# short" %}
7353
7354 ins_encode(aarch64_enc_ldarhw(dst, mem));
7355
7356 ins_pipe(pipe_serial);
7357 %}
7358
7359 // Load Short/Char (16 bit unsigned) into long
7360 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7361 %{
7362 match(Set dst (ConvI2L (LoadUS mem)));
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarh $dst, $mem\t# short" %}
7366
7367 ins_encode(aarch64_enc_ldarh(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 // Load Short/Char (16 bit signed) into long
7373 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7374 %{
7375 match(Set dst (ConvI2L (LoadS mem)));
7376
7377 ins_cost(VOLATILE_REF_COST);
7378 format %{ "ldarh $dst, $mem\t# short" %}
7379
7380 ins_encode(aarch64_enc_ldarsh(dst, mem));
7381
7382 ins_pipe(pipe_serial);
7383 %}
7384
7385 // Load Integer (32 bit signed)
7386 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7387 %{
7388 match(Set dst (LoadI mem));
7389
7390 ins_cost(VOLATILE_REF_COST);
7391 format %{ "ldarw $dst, $mem\t# int" %}
7392
7393 ins_encode(aarch64_enc_ldarw(dst, mem));
7394
7395 ins_pipe(pipe_serial);
7396 %}
7397
7398 // Load Integer (32 bit unsigned) into long
7399 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7400 %{
7401 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7402
7403 ins_cost(VOLATILE_REF_COST);
7404 format %{ "ldarw $dst, $mem\t# int" %}
7405
7406 ins_encode(aarch64_enc_ldarw(dst, mem));
7407
7408 ins_pipe(pipe_serial);
7409 %}
7410
7411 // Load Long (64 bit signed)
7412 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7413 %{
7414 match(Set dst (LoadL mem));
7415
7416 ins_cost(VOLATILE_REF_COST);
7417 format %{ "ldar $dst, $mem\t# int" %}
7418
7419 ins_encode(aarch64_enc_ldar(dst, mem));
7420
7421 ins_pipe(pipe_serial);
7422 %}
7423
7424 // Load Pointer
7425 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7426 %{
7427 match(Set dst (LoadP mem));
7428 predicate(n->as_Load()->barrier_data() == 0);
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "ldar $dst, $mem\t# ptr" %}
7432
7433 ins_encode(aarch64_enc_ldar(dst, mem));
7434
7435 ins_pipe(pipe_serial);
7436 %}
7437
7438 // Load Compressed Pointer
7439 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7440 %{
7441 match(Set dst (LoadN mem));
7442 predicate(n->as_Load()->barrier_data() == 0);
7443
7444 ins_cost(VOLATILE_REF_COST);
7445 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7446
7447 ins_encode(aarch64_enc_ldarw(dst, mem));
7448
7449 ins_pipe(pipe_serial);
7450 %}
7451
7452 // Load Float
7453 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7454 %{
7455 match(Set dst (LoadF mem));
7456
7457 ins_cost(VOLATILE_REF_COST);
7458 format %{ "ldars $dst, $mem\t# float" %}
7459
7460 ins_encode( aarch64_enc_fldars(dst, mem) );
7461
7462 ins_pipe(pipe_serial);
7463 %}
7464
7465 // Load Double
7466 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7467 %{
7468 match(Set dst (LoadD mem));
7469
7470 ins_cost(VOLATILE_REF_COST);
7471 format %{ "ldard $dst, $mem\t# double" %}
7472
7473 ins_encode( aarch64_enc_fldard(dst, mem) );
7474
7475 ins_pipe(pipe_serial);
7476 %}
7477
7478 // Store Byte
7479 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7480 %{
7481 match(Set mem (StoreB mem src));
7482
7483 ins_cost(VOLATILE_REF_COST);
7484 format %{ "stlrb $src, $mem\t# byte" %}
7485
7486 ins_encode(aarch64_enc_stlrb(src, mem));
7487
7488 ins_pipe(pipe_class_memory);
7489 %}
7490
7491 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7492 %{
7493 match(Set mem (StoreB mem zero));
7494
7495 ins_cost(VOLATILE_REF_COST);
7496 format %{ "stlrb zr, $mem\t# byte" %}
7497
7498 ins_encode(aarch64_enc_stlrb0(mem));
7499
7500 ins_pipe(pipe_class_memory);
7501 %}
7502
7503 // Store Char/Short
7504 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7505 %{
7506 match(Set mem (StoreC mem src));
7507
7508 ins_cost(VOLATILE_REF_COST);
7509 format %{ "stlrh $src, $mem\t# short" %}
7510
7511 ins_encode(aarch64_enc_stlrh(src, mem));
7512
7513 ins_pipe(pipe_class_memory);
7514 %}
7515
7516 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7517 %{
7518 match(Set mem (StoreC mem zero));
7519
7520 ins_cost(VOLATILE_REF_COST);
7521 format %{ "stlrh zr, $mem\t# short" %}
7522
7523 ins_encode(aarch64_enc_stlrh0(mem));
7524
7525 ins_pipe(pipe_class_memory);
7526 %}
7527
7528 // Store Integer
7529
7530 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7531 %{
7532 match(Set mem(StoreI mem src));
7533
7534 ins_cost(VOLATILE_REF_COST);
7535 format %{ "stlrw $src, $mem\t# int" %}
7536
7537 ins_encode(aarch64_enc_stlrw(src, mem));
7538
7539 ins_pipe(pipe_class_memory);
7540 %}
7541
7542 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7543 %{
7544 match(Set mem(StoreI mem zero));
7545
7546 ins_cost(VOLATILE_REF_COST);
7547 format %{ "stlrw zr, $mem\t# int" %}
7548
7549 ins_encode(aarch64_enc_stlrw0(mem));
7550
7551 ins_pipe(pipe_class_memory);
7552 %}
7553
7554 // Store Long (64 bit signed)
7555 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7556 %{
7557 match(Set mem (StoreL mem src));
7558
7559 ins_cost(VOLATILE_REF_COST);
7560 format %{ "stlr $src, $mem\t# int" %}
7561
7562 ins_encode(aarch64_enc_stlr(src, mem));
7563
7564 ins_pipe(pipe_class_memory);
7565 %}
7566
7567 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7568 %{
7569 match(Set mem (StoreL mem zero));
7570
7571 ins_cost(VOLATILE_REF_COST);
7572 format %{ "stlr zr, $mem\t# int" %}
7573
7574 ins_encode(aarch64_enc_stlr0(mem));
7575
7576 ins_pipe(pipe_class_memory);
7577 %}
7578
7579 // Store Pointer
7580 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7581 %{
7582 match(Set mem (StoreP mem src));
7583 predicate(n->as_Store()->barrier_data() == 0);
7584
7585 ins_cost(VOLATILE_REF_COST);
7586 format %{ "stlr $src, $mem\t# ptr" %}
7587
7588 ins_encode(aarch64_enc_stlr(src, mem));
7589
7590 ins_pipe(pipe_class_memory);
7591 %}
7592
7593 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7594 %{
7595 match(Set mem (StoreP mem zero));
7596 predicate(n->as_Store()->barrier_data() == 0);
7597
7598 ins_cost(VOLATILE_REF_COST);
7599 format %{ "stlr zr, $mem\t# ptr" %}
7600
7601 ins_encode(aarch64_enc_stlr0(mem));
7602
7603 ins_pipe(pipe_class_memory);
7604 %}
7605
7606 // Store Compressed Pointer
7607 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7608 %{
7609 match(Set mem (StoreN mem src));
7610 predicate(n->as_Store()->barrier_data() == 0);
7611
7612 ins_cost(VOLATILE_REF_COST);
7613 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7614
7615 ins_encode(aarch64_enc_stlrw(src, mem));
7616
7617 ins_pipe(pipe_class_memory);
7618 %}
7619
7620 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7621 %{
7622 match(Set mem (StoreN mem zero));
7623 predicate(n->as_Store()->barrier_data() == 0);
7624
7625 ins_cost(VOLATILE_REF_COST);
7626 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7627
7628 ins_encode(aarch64_enc_stlrw0(mem));
7629
7630 ins_pipe(pipe_class_memory);
7631 %}
7632
7633 // Store Float
7634 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7635 %{
7636 match(Set mem (StoreF mem src));
7637
7638 ins_cost(VOLATILE_REF_COST);
7639 format %{ "stlrs $src, $mem\t# float" %}
7640
7641 ins_encode( aarch64_enc_fstlrs(src, mem) );
7642
7643 ins_pipe(pipe_class_memory);
7644 %}
7645
7646 // TODO
7647 // implement storeImmF0 and storeFImmPacked
7648
7649 // Store Double
7650 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7651 %{
7652 match(Set mem (StoreD mem src));
7653
7654 ins_cost(VOLATILE_REF_COST);
7655 format %{ "stlrd $src, $mem\t# double" %}
7656
7657 ins_encode( aarch64_enc_fstlrd(src, mem) );
7658
7659 ins_pipe(pipe_class_memory);
7660 %}
7661
7662 // ---------------- end of volatile loads and stores ----------------
7663
7664 instruct cacheWB(indirect addr)
7665 %{
7666 predicate(VM_Version::supports_data_cache_line_flush());
7667 match(CacheWB addr);
7668
7669 ins_cost(100);
7670 format %{"cache wb $addr" %}
7671 ins_encode %{
7672 assert($addr->index_position() < 0, "should be");
7673 assert($addr$$disp == 0, "should be");
7674 __ cache_wb(Address($addr$$base$$Register, 0));
7675 %}
7676 ins_pipe(pipe_slow); // XXX
7677 %}
7678
7679 instruct cacheWBPreSync()
7680 %{
7681 predicate(VM_Version::supports_data_cache_line_flush());
7682 match(CacheWBPreSync);
7683
7684 ins_cost(100);
7685 format %{"cache wb presync" %}
7686 ins_encode %{
7687 __ cache_wbsync(true);
7688 %}
7689 ins_pipe(pipe_slow); // XXX
7690 %}
7691
7692 instruct cacheWBPostSync()
7693 %{
7694 predicate(VM_Version::supports_data_cache_line_flush());
7695 match(CacheWBPostSync);
7696
7697 ins_cost(100);
7698 format %{"cache wb postsync" %}
7699 ins_encode %{
7700 __ cache_wbsync(false);
7701 %}
7702 ins_pipe(pipe_slow); // XXX
7703 %}
7704
7705 // ============================================================================
7706 // BSWAP Instructions
7707
7708 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7709 match(Set dst (ReverseBytesI src));
7710
7711 ins_cost(INSN_COST);
7712 format %{ "revw $dst, $src" %}
7713
7714 ins_encode %{
7715 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7716 %}
7717
7718 ins_pipe(ialu_reg);
7719 %}
7720
7721 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7722 match(Set dst (ReverseBytesL src));
7723
7724 ins_cost(INSN_COST);
7725 format %{ "rev $dst, $src" %}
7726
7727 ins_encode %{
7728 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7729 %}
7730
7731 ins_pipe(ialu_reg);
7732 %}
7733
7734 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7735 match(Set dst (ReverseBytesUS src));
7736
7737 ins_cost(INSN_COST);
7738 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7739
7740 ins_encode %{
7741 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7742 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7743 %}
7744
7745 ins_pipe(ialu_reg);
7746 %}
7747
7748 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7749 match(Set dst (ReverseBytesS src));
7750
7751 ins_cost(INSN_COST);
7752 format %{ "rev16w $dst, $src\n\t"
7753 "sbfmw $dst, $dst, #0, #15" %}
7754
7755 ins_encode %{
7756 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7757 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7758 %}
7759
7760 ins_pipe(ialu_reg);
7761 %}
7762
7763 // ============================================================================
7764 // Zero Count Instructions
7765
7766 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7767 match(Set dst (CountLeadingZerosI src));
7768
7769 ins_cost(INSN_COST);
7770 format %{ "clzw $dst, $src" %}
7771 ins_encode %{
7772 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7773 %}
7774
7775 ins_pipe(ialu_reg);
7776 %}
7777
7778 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7779 match(Set dst (CountLeadingZerosL src));
7780
7781 ins_cost(INSN_COST);
7782 format %{ "clz $dst, $src" %}
7783 ins_encode %{
7784 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7785 %}
7786
7787 ins_pipe(ialu_reg);
7788 %}
7789
7790 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7791 match(Set dst (CountTrailingZerosI src));
7792
7793 ins_cost(INSN_COST * 2);
7794 format %{ "rbitw $dst, $src\n\t"
7795 "clzw $dst, $dst" %}
7796 ins_encode %{
7797 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7798 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7799 %}
7800
7801 ins_pipe(ialu_reg);
7802 %}
7803
7804 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7805 match(Set dst (CountTrailingZerosL src));
7806
7807 ins_cost(INSN_COST * 2);
7808 format %{ "rbit $dst, $src\n\t"
7809 "clz $dst, $dst" %}
7810 ins_encode %{
7811 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7812 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7813 %}
7814
7815 ins_pipe(ialu_reg);
7816 %}
7817
7818 //---------- Population Count Instructions -------------------------------------
7819 //
7820
7821 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7822 match(Set dst (PopCountI src));
7823 effect(TEMP tmp);
7824 ins_cost(INSN_COST * 13);
7825
7826 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7827 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7828 "addv $tmp, $tmp\t# vector (8B)\n\t"
7829 "mov $dst, $tmp\t# vector (1D)" %}
7830 ins_encode %{
7831 __ fmovs($tmp$$FloatRegister, $src$$Register);
7832 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7833 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7834 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7835 %}
7836
7837 ins_pipe(pipe_class_default);
7838 %}
7839
7840 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7841 match(Set dst (PopCountI (LoadI mem)));
7842 effect(TEMP tmp);
7843 ins_cost(INSN_COST * 13);
7844
7845 format %{ "ldrs $tmp, $mem\n\t"
7846 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7847 "addv $tmp, $tmp\t# vector (8B)\n\t"
7848 "mov $dst, $tmp\t# vector (1D)" %}
7849 ins_encode %{
7850 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7851 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7852 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7853 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7854 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7855 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7856 %}
7857
7858 ins_pipe(pipe_class_default);
7859 %}
7860
7861 // Note: Long.bitCount(long) returns an int.
7862 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7863 match(Set dst (PopCountL src));
7864 effect(TEMP tmp);
7865 ins_cost(INSN_COST * 13);
7866
7867 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7868 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7869 "addv $tmp, $tmp\t# vector (8B)\n\t"
7870 "mov $dst, $tmp\t# vector (1D)" %}
7871 ins_encode %{
7872 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7873 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7874 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7875 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7876 %}
7877
7878 ins_pipe(pipe_class_default);
7879 %}
7880
7881 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7882 match(Set dst (PopCountL (LoadL mem)));
7883 effect(TEMP tmp);
7884 ins_cost(INSN_COST * 13);
7885
7886 format %{ "ldrd $tmp, $mem\n\t"
7887 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7888 "addv $tmp, $tmp\t# vector (8B)\n\t"
7889 "mov $dst, $tmp\t# vector (1D)" %}
7890 ins_encode %{
7891 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7892 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7893 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7894 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7895 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7896 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7897 %}
7898
7899 ins_pipe(pipe_class_default);
7900 %}
7901
7902 // ============================================================================
7903 // VerifyVectorAlignment Instruction
7904
7905 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7906 match(Set addr (VerifyVectorAlignment addr mask));
7907 effect(KILL cr);
7908 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7909 ins_encode %{
7910 Label Lskip;
7911 // check if masked bits of addr are zero
7912 __ tst($addr$$Register, $mask$$constant);
7913 __ br(Assembler::EQ, Lskip);
7914 __ stop("verify_vector_alignment found a misaligned vector memory access");
7915 __ bind(Lskip);
7916 %}
7917 ins_pipe(pipe_slow);
7918 %}
7919
7920 // ============================================================================
7921 // MemBar Instruction
7922
7923 instruct load_fence() %{
7924 match(LoadFence);
7925 ins_cost(VOLATILE_REF_COST);
7926
7927 format %{ "load_fence" %}
7928
7929 ins_encode %{
7930 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7931 %}
7932 ins_pipe(pipe_serial);
7933 %}
7934
7935 instruct unnecessary_membar_acquire() %{
7936 predicate(unnecessary_acquire(n));
7937 match(MemBarAcquire);
7938 ins_cost(0);
7939
7940 format %{ "membar_acquire (elided)" %}
7941
7942 ins_encode %{
7943 __ block_comment("membar_acquire (elided)");
7944 %}
7945
7946 ins_pipe(pipe_class_empty);
7947 %}
7948
7949 instruct membar_acquire() %{
7950 match(MemBarAcquire);
7951 ins_cost(VOLATILE_REF_COST);
7952
7953 format %{ "membar_acquire\n\t"
7954 "dmb ishld" %}
7955
7956 ins_encode %{
7957 __ block_comment("membar_acquire");
7958 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7959 %}
7960
7961 ins_pipe(pipe_serial);
7962 %}
7963
7964
7965 instruct membar_acquire_lock() %{
7966 match(MemBarAcquireLock);
7967 ins_cost(VOLATILE_REF_COST);
7968
7969 format %{ "membar_acquire_lock (elided)" %}
7970
7971 ins_encode %{
7972 __ block_comment("membar_acquire_lock (elided)");
7973 %}
7974
7975 ins_pipe(pipe_serial);
7976 %}
7977
7978 instruct store_fence() %{
7979 match(StoreFence);
7980 ins_cost(VOLATILE_REF_COST);
7981
7982 format %{ "store_fence" %}
7983
7984 ins_encode %{
7985 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7986 %}
7987 ins_pipe(pipe_serial);
7988 %}
7989
7990 instruct unnecessary_membar_release() %{
7991 predicate(unnecessary_release(n));
7992 match(MemBarRelease);
7993 ins_cost(0);
7994
7995 format %{ "membar_release (elided)" %}
7996
7997 ins_encode %{
7998 __ block_comment("membar_release (elided)");
7999 %}
8000 ins_pipe(pipe_serial);
8001 %}
8002
8003 instruct membar_release() %{
8004 match(MemBarRelease);
8005 ins_cost(VOLATILE_REF_COST);
8006
8007 format %{ "membar_release\n\t"
8008 "dmb ishst\n\tdmb ishld" %}
8009
8010 ins_encode %{
8011 __ block_comment("membar_release");
8012 // These will be merged if AlwaysMergeDMB is enabled.
8013 __ membar(Assembler::StoreStore);
8014 __ membar(Assembler::LoadStore);
8015 %}
8016 ins_pipe(pipe_serial);
8017 %}
8018
8019 instruct membar_storestore() %{
8020 match(MemBarStoreStore);
8021 match(StoreStoreFence);
8022 ins_cost(VOLATILE_REF_COST);
8023
8024 format %{ "MEMBAR-store-store" %}
8025
8026 ins_encode %{
8027 __ membar(Assembler::StoreStore);
8028 %}
8029 ins_pipe(pipe_serial);
8030 %}
8031
8032 instruct membar_release_lock() %{
8033 match(MemBarReleaseLock);
8034 ins_cost(VOLATILE_REF_COST);
8035
8036 format %{ "membar_release_lock (elided)" %}
8037
8038 ins_encode %{
8039 __ block_comment("membar_release_lock (elided)");
8040 %}
8041
8042 ins_pipe(pipe_serial);
8043 %}
8044
8045 instruct membar_storeload() %{
8046 match(MemBarStoreLoad);
8047 ins_cost(VOLATILE_REF_COST*100);
8048
8049 format %{ "MEMBAR-store-load\n\t"
8050 "dmb ish" %}
8051
8052 ins_encode %{
8053 __ block_comment("membar_storeload");
8054 __ membar(Assembler::StoreLoad);
8055 %}
8056
8057 ins_pipe(pipe_serial);
8058 %}
8059
8060 instruct unnecessary_membar_volatile() %{
8061 predicate(unnecessary_volatile(n));
8062 match(MemBarVolatile);
8063 ins_cost(0);
8064
8065 format %{ "membar_volatile (elided)" %}
8066
8067 ins_encode %{
8068 __ block_comment("membar_volatile (elided)");
8069 %}
8070
8071 ins_pipe(pipe_serial);
8072 %}
8073
8074 instruct membar_volatile() %{
8075 match(MemBarVolatile);
8076 ins_cost(VOLATILE_REF_COST*100);
8077
8078 format %{ "membar_volatile\n\t"
8079 "dmb ish"%}
8080
8081 ins_encode %{
8082 __ block_comment("membar_volatile");
8083 __ membar(Assembler::StoreLoad);
8084 %}
8085
8086 ins_pipe(pipe_serial);
8087 %}
8088
8089 instruct membar_full() %{
8090 match(MemBarFull);
8091 ins_cost(VOLATILE_REF_COST*100);
8092
8093 format %{ "membar_full\n\t"
8094 "dmb ish" %}
8095 ins_encode %{
8096 __ block_comment("membar_full");
8097 __ membar(Assembler::AnyAny);
8098 %}
8099
8100 ins_pipe(pipe_serial);
8101 %}
8102
8103 // ============================================================================
8104 // Cast/Convert Instructions
8105
8106 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8107 match(Set dst (CastX2P src));
8108
8109 ins_cost(INSN_COST);
8110 format %{ "mov $dst, $src\t# long -> ptr" %}
8111
8112 ins_encode %{
8113 if ($dst$$reg != $src$$reg) {
8114 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8115 }
8116 %}
8117
8118 ins_pipe(ialu_reg);
8119 %}
8120
8121 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8122 match(Set dst (CastP2X src));
8123
8124 ins_cost(INSN_COST);
8125 format %{ "mov $dst, $src\t# ptr -> long" %}
8126
8127 ins_encode %{
8128 if ($dst$$reg != $src$$reg) {
8129 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8130 }
8131 %}
8132
8133 ins_pipe(ialu_reg);
8134 %}
8135
8136 // Convert oop into int for vectors alignment masking
8137 instruct convP2I(iRegINoSp dst, iRegP src) %{
8138 match(Set dst (ConvL2I (CastP2X src)));
8139
8140 ins_cost(INSN_COST);
8141 format %{ "movw $dst, $src\t# ptr -> int" %}
8142 ins_encode %{
8143 __ movw($dst$$Register, $src$$Register);
8144 %}
8145
8146 ins_pipe(ialu_reg);
8147 %}
8148
8149 // Convert compressed oop into int for vectors alignment masking
8150 // in case of 32bit oops (heap < 4Gb).
8151 instruct convN2I(iRegINoSp dst, iRegN src)
8152 %{
8153 predicate(CompressedOops::shift() == 0);
8154 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8155
8156 ins_cost(INSN_COST);
8157 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8158 ins_encode %{
8159 __ movw($dst$$Register, $src$$Register);
8160 %}
8161
8162 ins_pipe(ialu_reg);
8163 %}
8164
8165
8166 // Convert oop pointer into compressed form
8167 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8168 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8169 match(Set dst (EncodeP src));
8170 effect(KILL cr);
8171 ins_cost(INSN_COST * 3);
8172 format %{ "encode_heap_oop $dst, $src" %}
8173 ins_encode %{
8174 Register s = $src$$Register;
8175 Register d = $dst$$Register;
8176 __ encode_heap_oop(d, s);
8177 %}
8178 ins_pipe(ialu_reg);
8179 %}
8180
8181 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8182 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8183 match(Set dst (EncodeP src));
8184 ins_cost(INSN_COST * 3);
8185 format %{ "encode_heap_oop_not_null $dst, $src" %}
8186 ins_encode %{
8187 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8188 %}
8189 ins_pipe(ialu_reg);
8190 %}
8191
8192 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8193 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8194 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8195 match(Set dst (DecodeN src));
8196 ins_cost(INSN_COST * 3);
8197 format %{ "decode_heap_oop $dst, $src" %}
8198 ins_encode %{
8199 Register s = $src$$Register;
8200 Register d = $dst$$Register;
8201 __ decode_heap_oop(d, s);
8202 %}
8203 ins_pipe(ialu_reg);
8204 %}
8205
8206 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8207 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8208 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8209 match(Set dst (DecodeN src));
8210 ins_cost(INSN_COST * 3);
8211 format %{ "decode_heap_oop_not_null $dst, $src" %}
8212 ins_encode %{
8213 Register s = $src$$Register;
8214 Register d = $dst$$Register;
8215 __ decode_heap_oop_not_null(d, s);
8216 %}
8217 ins_pipe(ialu_reg);
8218 %}
8219
8220 // n.b. AArch64 implementations of encode_klass_not_null and
8221 // decode_klass_not_null do not modify the flags register so, unlike
8222 // Intel, we don't kill CR as a side effect here
8223
8224 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8225 match(Set dst (EncodePKlass src));
8226
8227 ins_cost(INSN_COST * 3);
8228 format %{ "encode_klass_not_null $dst,$src" %}
8229
8230 ins_encode %{
8231 Register src_reg = as_Register($src$$reg);
8232 Register dst_reg = as_Register($dst$$reg);
8233 __ encode_klass_not_null(dst_reg, src_reg);
8234 %}
8235
8236 ins_pipe(ialu_reg);
8237 %}
8238
8239 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8240 match(Set dst (DecodeNKlass src));
8241
8242 ins_cost(INSN_COST * 3);
8243 format %{ "decode_klass_not_null $dst,$src" %}
8244
8245 ins_encode %{
8246 Register src_reg = as_Register($src$$reg);
8247 Register dst_reg = as_Register($dst$$reg);
8248 if (dst_reg != src_reg) {
8249 __ decode_klass_not_null(dst_reg, src_reg);
8250 } else {
8251 __ decode_klass_not_null(dst_reg);
8252 }
8253 %}
8254
8255 ins_pipe(ialu_reg);
8256 %}
8257
8258 instruct checkCastPP(iRegPNoSp dst)
8259 %{
8260 match(Set dst (CheckCastPP dst));
8261
8262 size(0);
8263 format %{ "# checkcastPP of $dst" %}
8264 ins_encode(/* empty encoding */);
8265 ins_pipe(pipe_class_empty);
8266 %}
8267
8268 instruct castPP(iRegPNoSp dst)
8269 %{
8270 match(Set dst (CastPP dst));
8271
8272 size(0);
8273 format %{ "# castPP of $dst" %}
8274 ins_encode(/* empty encoding */);
8275 ins_pipe(pipe_class_empty);
8276 %}
8277
8278 instruct castII(iRegI dst)
8279 %{
8280 predicate(VerifyConstraintCasts == 0);
8281 match(Set dst (CastII dst));
8282
8283 size(0);
8284 format %{ "# castII of $dst" %}
8285 ins_encode(/* empty encoding */);
8286 ins_cost(0);
8287 ins_pipe(pipe_class_empty);
8288 %}
8289
8290 instruct castII_checked(iRegI dst, rFlagsReg cr)
8291 %{
8292 predicate(VerifyConstraintCasts > 0);
8293 match(Set dst (CastII dst));
8294 effect(KILL cr);
8295
8296 format %{ "# castII_checked of $dst" %}
8297 ins_encode %{
8298 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8299 %}
8300 ins_pipe(pipe_slow);
8301 %}
8302
8303 instruct castLL(iRegL dst)
8304 %{
8305 predicate(VerifyConstraintCasts == 0);
8306 match(Set dst (CastLL dst));
8307
8308 size(0);
8309 format %{ "# castLL of $dst" %}
8310 ins_encode(/* empty encoding */);
8311 ins_cost(0);
8312 ins_pipe(pipe_class_empty);
8313 %}
8314
8315 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8316 %{
8317 predicate(VerifyConstraintCasts > 0);
8318 match(Set dst (CastLL dst));
8319 effect(KILL cr);
8320
8321 format %{ "# castLL_checked of $dst" %}
8322 ins_encode %{
8323 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8324 %}
8325 ins_pipe(pipe_slow);
8326 %}
8327
8328 instruct castHH(vRegF dst)
8329 %{
8330 match(Set dst (CastHH dst));
8331 size(0);
8332 format %{ "# castHH of $dst" %}
8333 ins_encode(/* empty encoding */);
8334 ins_cost(0);
8335 ins_pipe(pipe_class_empty);
8336 %}
8337
8338 instruct castFF(vRegF dst)
8339 %{
8340 match(Set dst (CastFF dst));
8341
8342 size(0);
8343 format %{ "# castFF of $dst" %}
8344 ins_encode(/* empty encoding */);
8345 ins_cost(0);
8346 ins_pipe(pipe_class_empty);
8347 %}
8348
8349 instruct castDD(vRegD dst)
8350 %{
8351 match(Set dst (CastDD dst));
8352
8353 size(0);
8354 format %{ "# castDD of $dst" %}
8355 ins_encode(/* empty encoding */);
8356 ins_cost(0);
8357 ins_pipe(pipe_class_empty);
8358 %}
8359
8360 instruct castVV(vReg dst)
8361 %{
8362 match(Set dst (CastVV dst));
8363
8364 size(0);
8365 format %{ "# castVV of $dst" %}
8366 ins_encode(/* empty encoding */);
8367 ins_cost(0);
8368 ins_pipe(pipe_class_empty);
8369 %}
8370
8371 instruct castVVMask(pRegGov dst)
8372 %{
8373 match(Set dst (CastVV dst));
8374
8375 size(0);
8376 format %{ "# castVV of $dst" %}
8377 ins_encode(/* empty encoding */);
8378 ins_cost(0);
8379 ins_pipe(pipe_class_empty);
8380 %}
8381
8382 // Manifest a CmpU result in an integer register.
8383 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8384 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8385 %{
8386 match(Set dst (CmpU3 src1 src2));
8387 effect(KILL flags);
8388
8389 ins_cost(INSN_COST * 3);
8390 format %{
8391 "cmpw $src1, $src2\n\t"
8392 "csetw $dst, ne\n\t"
8393 "cnegw $dst, lo\t# CmpU3(reg)"
8394 %}
8395 ins_encode %{
8396 __ cmpw($src1$$Register, $src2$$Register);
8397 __ csetw($dst$$Register, Assembler::NE);
8398 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8399 %}
8400
8401 ins_pipe(pipe_class_default);
8402 %}
8403
8404 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8405 %{
8406 match(Set dst (CmpU3 src1 src2));
8407 effect(KILL flags);
8408
8409 ins_cost(INSN_COST * 3);
8410 format %{
8411 "subsw zr, $src1, $src2\n\t"
8412 "csetw $dst, ne\n\t"
8413 "cnegw $dst, lo\t# CmpU3(imm)"
8414 %}
8415 ins_encode %{
8416 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8417 __ csetw($dst$$Register, Assembler::NE);
8418 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8419 %}
8420
8421 ins_pipe(pipe_class_default);
8422 %}
8423
8424 // Manifest a CmpUL result in an integer register.
8425 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8426 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8427 %{
8428 match(Set dst (CmpUL3 src1 src2));
8429 effect(KILL flags);
8430
8431 ins_cost(INSN_COST * 3);
8432 format %{
8433 "cmp $src1, $src2\n\t"
8434 "csetw $dst, ne\n\t"
8435 "cnegw $dst, lo\t# CmpUL3(reg)"
8436 %}
8437 ins_encode %{
8438 __ cmp($src1$$Register, $src2$$Register);
8439 __ csetw($dst$$Register, Assembler::NE);
8440 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8441 %}
8442
8443 ins_pipe(pipe_class_default);
8444 %}
8445
8446 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8447 %{
8448 match(Set dst (CmpUL3 src1 src2));
8449 effect(KILL flags);
8450
8451 ins_cost(INSN_COST * 3);
8452 format %{
8453 "subs zr, $src1, $src2\n\t"
8454 "csetw $dst, ne\n\t"
8455 "cnegw $dst, lo\t# CmpUL3(imm)"
8456 %}
8457 ins_encode %{
8458 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8459 __ csetw($dst$$Register, Assembler::NE);
8460 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8461 %}
8462
8463 ins_pipe(pipe_class_default);
8464 %}
8465
8466 // Manifest a CmpL result in an integer register.
8467 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8468 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8469 %{
8470 match(Set dst (CmpL3 src1 src2));
8471 effect(KILL flags);
8472
8473 ins_cost(INSN_COST * 3);
8474 format %{
8475 "cmp $src1, $src2\n\t"
8476 "csetw $dst, ne\n\t"
8477 "cnegw $dst, lt\t# CmpL3(reg)"
8478 %}
8479 ins_encode %{
8480 __ cmp($src1$$Register, $src2$$Register);
8481 __ csetw($dst$$Register, Assembler::NE);
8482 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8483 %}
8484
8485 ins_pipe(pipe_class_default);
8486 %}
8487
8488 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8489 %{
8490 match(Set dst (CmpL3 src1 src2));
8491 effect(KILL flags);
8492
8493 ins_cost(INSN_COST * 3);
8494 format %{
8495 "subs zr, $src1, $src2\n\t"
8496 "csetw $dst, ne\n\t"
8497 "cnegw $dst, lt\t# CmpL3(imm)"
8498 %}
8499 ins_encode %{
8500 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8501 __ csetw($dst$$Register, Assembler::NE);
8502 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8503 %}
8504
8505 ins_pipe(pipe_class_default);
8506 %}
8507
8508 // ============================================================================
8509 // Conditional Move Instructions
8510
8511 // n.b. we have identical rules for both a signed compare op (cmpOp)
8512 // and an unsigned compare op (cmpOpU). it would be nice if we could
8513 // define an op class which merged both inputs and use it to type the
8514 // argument to a single rule. unfortunatelyt his fails because the
8515 // opclass does not live up to the COND_INTER interface of its
8516 // component operands. When the generic code tries to negate the
8517 // operand it ends up running the generci Machoper::negate method
8518 // which throws a ShouldNotHappen. So, we have to provide two flavours
8519 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8520
8521 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8522 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8523
8524 ins_cost(INSN_COST * 2);
8525 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8526
8527 ins_encode %{
8528 __ cselw(as_Register($dst$$reg),
8529 as_Register($src2$$reg),
8530 as_Register($src1$$reg),
8531 (Assembler::Condition)$cmp$$cmpcode);
8532 %}
8533
8534 ins_pipe(icond_reg_reg);
8535 %}
8536
8537 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8538 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8539
8540 ins_cost(INSN_COST * 2);
8541 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8542
8543 ins_encode %{
8544 __ cselw(as_Register($dst$$reg),
8545 as_Register($src2$$reg),
8546 as_Register($src1$$reg),
8547 (Assembler::Condition)$cmp$$cmpcode);
8548 %}
8549
8550 ins_pipe(icond_reg_reg);
8551 %}
8552
8553 // special cases where one arg is zero
8554
8555 // n.b. this is selected in preference to the rule above because it
8556 // avoids loading constant 0 into a source register
8557
8558 // TODO
8559 // we ought only to be able to cull one of these variants as the ideal
8560 // transforms ought always to order the zero consistently (to left/right?)
8561
8562 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8563 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8564
8565 ins_cost(INSN_COST * 2);
8566 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8567
8568 ins_encode %{
8569 __ cselw(as_Register($dst$$reg),
8570 as_Register($src$$reg),
8571 zr,
8572 (Assembler::Condition)$cmp$$cmpcode);
8573 %}
8574
8575 ins_pipe(icond_reg);
8576 %}
8577
8578 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8579 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8580
8581 ins_cost(INSN_COST * 2);
8582 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8583
8584 ins_encode %{
8585 __ cselw(as_Register($dst$$reg),
8586 as_Register($src$$reg),
8587 zr,
8588 (Assembler::Condition)$cmp$$cmpcode);
8589 %}
8590
8591 ins_pipe(icond_reg);
8592 %}
8593
8594 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8595 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8596
8597 ins_cost(INSN_COST * 2);
8598 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8599
8600 ins_encode %{
8601 __ cselw(as_Register($dst$$reg),
8602 zr,
8603 as_Register($src$$reg),
8604 (Assembler::Condition)$cmp$$cmpcode);
8605 %}
8606
8607 ins_pipe(icond_reg);
8608 %}
8609
8610 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8611 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8612
8613 ins_cost(INSN_COST * 2);
8614 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8615
8616 ins_encode %{
8617 __ cselw(as_Register($dst$$reg),
8618 zr,
8619 as_Register($src$$reg),
8620 (Assembler::Condition)$cmp$$cmpcode);
8621 %}
8622
8623 ins_pipe(icond_reg);
8624 %}
8625
8626 // special case for creating a boolean 0 or 1
8627
8628 // n.b. this is selected in preference to the rule above because it
8629 // avoids loading constants 0 and 1 into a source register
8630
8631 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8632 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8633
8634 ins_cost(INSN_COST * 2);
8635 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8636
8637 ins_encode %{
8638 // equivalently
8639 // cset(as_Register($dst$$reg),
8640 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8641 __ csincw(as_Register($dst$$reg),
8642 zr,
8643 zr,
8644 (Assembler::Condition)$cmp$$cmpcode);
8645 %}
8646
8647 ins_pipe(icond_none);
8648 %}
8649
8650 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8651 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8652
8653 ins_cost(INSN_COST * 2);
8654 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8655
8656 ins_encode %{
8657 // equivalently
8658 // cset(as_Register($dst$$reg),
8659 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8660 __ csincw(as_Register($dst$$reg),
8661 zr,
8662 zr,
8663 (Assembler::Condition)$cmp$$cmpcode);
8664 %}
8665
8666 ins_pipe(icond_none);
8667 %}
8668
8669 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8670 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8671
8672 ins_cost(INSN_COST * 2);
8673 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8674
8675 ins_encode %{
8676 __ csel(as_Register($dst$$reg),
8677 as_Register($src2$$reg),
8678 as_Register($src1$$reg),
8679 (Assembler::Condition)$cmp$$cmpcode);
8680 %}
8681
8682 ins_pipe(icond_reg_reg);
8683 %}
8684
8685 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8686 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8687
8688 ins_cost(INSN_COST * 2);
8689 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8690
8691 ins_encode %{
8692 __ csel(as_Register($dst$$reg),
8693 as_Register($src2$$reg),
8694 as_Register($src1$$reg),
8695 (Assembler::Condition)$cmp$$cmpcode);
8696 %}
8697
8698 ins_pipe(icond_reg_reg);
8699 %}
8700
8701 // special cases where one arg is zero
8702
8703 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8704 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8705
8706 ins_cost(INSN_COST * 2);
8707 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8708
8709 ins_encode %{
8710 __ csel(as_Register($dst$$reg),
8711 zr,
8712 as_Register($src$$reg),
8713 (Assembler::Condition)$cmp$$cmpcode);
8714 %}
8715
8716 ins_pipe(icond_reg);
8717 %}
8718
8719 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8720 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8721
8722 ins_cost(INSN_COST * 2);
8723 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8724
8725 ins_encode %{
8726 __ csel(as_Register($dst$$reg),
8727 zr,
8728 as_Register($src$$reg),
8729 (Assembler::Condition)$cmp$$cmpcode);
8730 %}
8731
8732 ins_pipe(icond_reg);
8733 %}
8734
8735 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8736 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8737
8738 ins_cost(INSN_COST * 2);
8739 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8740
8741 ins_encode %{
8742 __ csel(as_Register($dst$$reg),
8743 as_Register($src$$reg),
8744 zr,
8745 (Assembler::Condition)$cmp$$cmpcode);
8746 %}
8747
8748 ins_pipe(icond_reg);
8749 %}
8750
8751 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8752 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8753
8754 ins_cost(INSN_COST * 2);
8755 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8756
8757 ins_encode %{
8758 __ csel(as_Register($dst$$reg),
8759 as_Register($src$$reg),
8760 zr,
8761 (Assembler::Condition)$cmp$$cmpcode);
8762 %}
8763
8764 ins_pipe(icond_reg);
8765 %}
8766
8767 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8768 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8769
8770 ins_cost(INSN_COST * 2);
8771 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8772
8773 ins_encode %{
8774 __ csel(as_Register($dst$$reg),
8775 as_Register($src2$$reg),
8776 as_Register($src1$$reg),
8777 (Assembler::Condition)$cmp$$cmpcode);
8778 %}
8779
8780 ins_pipe(icond_reg_reg);
8781 %}
8782
8783 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8784 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8785
8786 ins_cost(INSN_COST * 2);
8787 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8788
8789 ins_encode %{
8790 __ csel(as_Register($dst$$reg),
8791 as_Register($src2$$reg),
8792 as_Register($src1$$reg),
8793 (Assembler::Condition)$cmp$$cmpcode);
8794 %}
8795
8796 ins_pipe(icond_reg_reg);
8797 %}
8798
8799 // special cases where one arg is zero
8800
8801 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8802 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8803
8804 ins_cost(INSN_COST * 2);
8805 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8806
8807 ins_encode %{
8808 __ csel(as_Register($dst$$reg),
8809 zr,
8810 as_Register($src$$reg),
8811 (Assembler::Condition)$cmp$$cmpcode);
8812 %}
8813
8814 ins_pipe(icond_reg);
8815 %}
8816
8817 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8818 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8819
8820 ins_cost(INSN_COST * 2);
8821 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8822
8823 ins_encode %{
8824 __ csel(as_Register($dst$$reg),
8825 zr,
8826 as_Register($src$$reg),
8827 (Assembler::Condition)$cmp$$cmpcode);
8828 %}
8829
8830 ins_pipe(icond_reg);
8831 %}
8832
8833 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8834 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8835
8836 ins_cost(INSN_COST * 2);
8837 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8838
8839 ins_encode %{
8840 __ csel(as_Register($dst$$reg),
8841 as_Register($src$$reg),
8842 zr,
8843 (Assembler::Condition)$cmp$$cmpcode);
8844 %}
8845
8846 ins_pipe(icond_reg);
8847 %}
8848
8849 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8850 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8851
8852 ins_cost(INSN_COST * 2);
8853 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8854
8855 ins_encode %{
8856 __ csel(as_Register($dst$$reg),
8857 as_Register($src$$reg),
8858 zr,
8859 (Assembler::Condition)$cmp$$cmpcode);
8860 %}
8861
8862 ins_pipe(icond_reg);
8863 %}
8864
8865 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8866 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8867
8868 ins_cost(INSN_COST * 2);
8869 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8870
8871 ins_encode %{
8872 __ cselw(as_Register($dst$$reg),
8873 as_Register($src2$$reg),
8874 as_Register($src1$$reg),
8875 (Assembler::Condition)$cmp$$cmpcode);
8876 %}
8877
8878 ins_pipe(icond_reg_reg);
8879 %}
8880
8881 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8882 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8883
8884 ins_cost(INSN_COST * 2);
8885 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8886
8887 ins_encode %{
8888 __ cselw(as_Register($dst$$reg),
8889 as_Register($src2$$reg),
8890 as_Register($src1$$reg),
8891 (Assembler::Condition)$cmp$$cmpcode);
8892 %}
8893
8894 ins_pipe(icond_reg_reg);
8895 %}
8896
8897 // special cases where one arg is zero
8898
8899 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8900 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8901
8902 ins_cost(INSN_COST * 2);
8903 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8904
8905 ins_encode %{
8906 __ cselw(as_Register($dst$$reg),
8907 zr,
8908 as_Register($src$$reg),
8909 (Assembler::Condition)$cmp$$cmpcode);
8910 %}
8911
8912 ins_pipe(icond_reg);
8913 %}
8914
8915 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8916 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8917
8918 ins_cost(INSN_COST * 2);
8919 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8920
8921 ins_encode %{
8922 __ cselw(as_Register($dst$$reg),
8923 zr,
8924 as_Register($src$$reg),
8925 (Assembler::Condition)$cmp$$cmpcode);
8926 %}
8927
8928 ins_pipe(icond_reg);
8929 %}
8930
8931 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8932 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8933
8934 ins_cost(INSN_COST * 2);
8935 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8936
8937 ins_encode %{
8938 __ cselw(as_Register($dst$$reg),
8939 as_Register($src$$reg),
8940 zr,
8941 (Assembler::Condition)$cmp$$cmpcode);
8942 %}
8943
8944 ins_pipe(icond_reg);
8945 %}
8946
8947 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8948 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8949
8950 ins_cost(INSN_COST * 2);
8951 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8952
8953 ins_encode %{
8954 __ cselw(as_Register($dst$$reg),
8955 as_Register($src$$reg),
8956 zr,
8957 (Assembler::Condition)$cmp$$cmpcode);
8958 %}
8959
8960 ins_pipe(icond_reg);
8961 %}
8962
8963 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8964 %{
8965 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8966
8967 ins_cost(INSN_COST * 3);
8968
8969 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8970 ins_encode %{
8971 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8972 __ fcsels(as_FloatRegister($dst$$reg),
8973 as_FloatRegister($src2$$reg),
8974 as_FloatRegister($src1$$reg),
8975 cond);
8976 %}
8977
8978 ins_pipe(fp_cond_reg_reg_s);
8979 %}
8980
8981 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8982 %{
8983 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8984
8985 ins_cost(INSN_COST * 3);
8986
8987 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8988 ins_encode %{
8989 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8990 __ fcsels(as_FloatRegister($dst$$reg),
8991 as_FloatRegister($src2$$reg),
8992 as_FloatRegister($src1$$reg),
8993 cond);
8994 %}
8995
8996 ins_pipe(fp_cond_reg_reg_s);
8997 %}
8998
8999 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9000 %{
9001 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9002
9003 ins_cost(INSN_COST * 3);
9004
9005 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9006 ins_encode %{
9007 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9008 __ fcseld(as_FloatRegister($dst$$reg),
9009 as_FloatRegister($src2$$reg),
9010 as_FloatRegister($src1$$reg),
9011 cond);
9012 %}
9013
9014 ins_pipe(fp_cond_reg_reg_d);
9015 %}
9016
9017 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9018 %{
9019 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9020
9021 ins_cost(INSN_COST * 3);
9022
9023 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9024 ins_encode %{
9025 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9026 __ fcseld(as_FloatRegister($dst$$reg),
9027 as_FloatRegister($src2$$reg),
9028 as_FloatRegister($src1$$reg),
9029 cond);
9030 %}
9031
9032 ins_pipe(fp_cond_reg_reg_d);
9033 %}
9034
9035 // ============================================================================
9036 // Arithmetic Instructions
9037 //
9038
9039 // Integer Addition
9040
9041 // TODO
9042 // these currently employ operations which do not set CR and hence are
9043 // not flagged as killing CR but we would like to isolate the cases
9044 // where we want to set flags from those where we don't. need to work
9045 // out how to do that.
9046
9047 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9048 match(Set dst (AddI src1 src2));
9049
9050 ins_cost(INSN_COST);
9051 format %{ "addw $dst, $src1, $src2" %}
9052
9053 ins_encode %{
9054 __ addw(as_Register($dst$$reg),
9055 as_Register($src1$$reg),
9056 as_Register($src2$$reg));
9057 %}
9058
9059 ins_pipe(ialu_reg_reg);
9060 %}
9061
9062 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9063 match(Set dst (AddI src1 src2));
9064
9065 ins_cost(INSN_COST);
9066 format %{ "addw $dst, $src1, $src2" %}
9067
9068 // use opcode to indicate that this is an add not a sub
9069 opcode(0x0);
9070
9071 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9072
9073 ins_pipe(ialu_reg_imm);
9074 %}
9075
9076 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9077 match(Set dst (AddI (ConvL2I src1) src2));
9078
9079 ins_cost(INSN_COST);
9080 format %{ "addw $dst, $src1, $src2" %}
9081
9082 // use opcode to indicate that this is an add not a sub
9083 opcode(0x0);
9084
9085 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9086
9087 ins_pipe(ialu_reg_imm);
9088 %}
9089
9090 // Pointer Addition
9091 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9092 match(Set dst (AddP src1 src2));
9093
9094 ins_cost(INSN_COST);
9095 format %{ "add $dst, $src1, $src2\t# ptr" %}
9096
9097 ins_encode %{
9098 __ add(as_Register($dst$$reg),
9099 as_Register($src1$$reg),
9100 as_Register($src2$$reg));
9101 %}
9102
9103 ins_pipe(ialu_reg_reg);
9104 %}
9105
9106 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9107 match(Set dst (AddP src1 (ConvI2L src2)));
9108
9109 ins_cost(1.9 * INSN_COST);
9110 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9111
9112 ins_encode %{
9113 __ add(as_Register($dst$$reg),
9114 as_Register($src1$$reg),
9115 as_Register($src2$$reg), ext::sxtw);
9116 %}
9117
9118 ins_pipe(ialu_reg_reg);
9119 %}
9120
9121 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9122 match(Set dst (AddP src1 (LShiftL src2 scale)));
9123
9124 ins_cost(1.9 * INSN_COST);
9125 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9126
9127 ins_encode %{
9128 __ lea(as_Register($dst$$reg),
9129 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9130 Address::lsl($scale$$constant)));
9131 %}
9132
9133 ins_pipe(ialu_reg_reg_shift);
9134 %}
9135
9136 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9137 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9138
9139 ins_cost(1.9 * INSN_COST);
9140 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9141
9142 ins_encode %{
9143 __ lea(as_Register($dst$$reg),
9144 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9145 Address::sxtw($scale$$constant)));
9146 %}
9147
9148 ins_pipe(ialu_reg_reg_shift);
9149 %}
9150
9151 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9152 match(Set dst (LShiftL (ConvI2L src) scale));
9153
9154 ins_cost(INSN_COST);
9155 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9156
9157 ins_encode %{
9158 __ sbfiz(as_Register($dst$$reg),
9159 as_Register($src$$reg),
9160 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9161 %}
9162
9163 ins_pipe(ialu_reg_shift);
9164 %}
9165
9166 // Pointer Immediate Addition
9167 // n.b. this needs to be more expensive than using an indirect memory
9168 // operand
9169 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9170 match(Set dst (AddP src1 src2));
9171
9172 ins_cost(INSN_COST);
9173 format %{ "add $dst, $src1, $src2\t# ptr" %}
9174
9175 // use opcode to indicate that this is an add not a sub
9176 opcode(0x0);
9177
9178 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9179
9180 ins_pipe(ialu_reg_imm);
9181 %}
9182
9183 // Long Addition
9184 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9185
9186 match(Set dst (AddL src1 src2));
9187
9188 ins_cost(INSN_COST);
9189 format %{ "add $dst, $src1, $src2" %}
9190
9191 ins_encode %{
9192 __ add(as_Register($dst$$reg),
9193 as_Register($src1$$reg),
9194 as_Register($src2$$reg));
9195 %}
9196
9197 ins_pipe(ialu_reg_reg);
9198 %}
9199
9200 // No constant pool entries requiredLong Immediate Addition.
9201 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9202 match(Set dst (AddL src1 src2));
9203
9204 ins_cost(INSN_COST);
9205 format %{ "add $dst, $src1, $src2" %}
9206
9207 // use opcode to indicate that this is an add not a sub
9208 opcode(0x0);
9209
9210 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9211
9212 ins_pipe(ialu_reg_imm);
9213 %}
9214
9215 // Integer Subtraction
9216 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9217 match(Set dst (SubI src1 src2));
9218
9219 ins_cost(INSN_COST);
9220 format %{ "subw $dst, $src1, $src2" %}
9221
9222 ins_encode %{
9223 __ subw(as_Register($dst$$reg),
9224 as_Register($src1$$reg),
9225 as_Register($src2$$reg));
9226 %}
9227
9228 ins_pipe(ialu_reg_reg);
9229 %}
9230
9231 // Immediate Subtraction
9232 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9233 match(Set dst (SubI src1 src2));
9234
9235 ins_cost(INSN_COST);
9236 format %{ "subw $dst, $src1, $src2" %}
9237
9238 // use opcode to indicate that this is a sub not an add
9239 opcode(0x1);
9240
9241 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9242
9243 ins_pipe(ialu_reg_imm);
9244 %}
9245
9246 // Long Subtraction
9247 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9248
9249 match(Set dst (SubL src1 src2));
9250
9251 ins_cost(INSN_COST);
9252 format %{ "sub $dst, $src1, $src2" %}
9253
9254 ins_encode %{
9255 __ sub(as_Register($dst$$reg),
9256 as_Register($src1$$reg),
9257 as_Register($src2$$reg));
9258 %}
9259
9260 ins_pipe(ialu_reg_reg);
9261 %}
9262
9263 // No constant pool entries requiredLong Immediate Subtraction.
9264 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9265 match(Set dst (SubL src1 src2));
9266
9267 ins_cost(INSN_COST);
9268 format %{ "sub$dst, $src1, $src2" %}
9269
9270 // use opcode to indicate that this is a sub not an add
9271 opcode(0x1);
9272
9273 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9274
9275 ins_pipe(ialu_reg_imm);
9276 %}
9277
9278 // Integer Negation (special case for sub)
9279
9280 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9281 match(Set dst (SubI zero src));
9282
9283 ins_cost(INSN_COST);
9284 format %{ "negw $dst, $src\t# int" %}
9285
9286 ins_encode %{
9287 __ negw(as_Register($dst$$reg),
9288 as_Register($src$$reg));
9289 %}
9290
9291 ins_pipe(ialu_reg);
9292 %}
9293
9294 // Long Negation
9295
9296 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9297 match(Set dst (SubL zero src));
9298
9299 ins_cost(INSN_COST);
9300 format %{ "neg $dst, $src\t# long" %}
9301
9302 ins_encode %{
9303 __ neg(as_Register($dst$$reg),
9304 as_Register($src$$reg));
9305 %}
9306
9307 ins_pipe(ialu_reg);
9308 %}
9309
9310 // Integer Multiply
9311
9312 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9313 match(Set dst (MulI src1 src2));
9314
9315 ins_cost(INSN_COST * 3);
9316 format %{ "mulw $dst, $src1, $src2" %}
9317
9318 ins_encode %{
9319 __ mulw(as_Register($dst$$reg),
9320 as_Register($src1$$reg),
9321 as_Register($src2$$reg));
9322 %}
9323
9324 ins_pipe(imul_reg_reg);
9325 %}
9326
9327 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9328 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9329
9330 ins_cost(INSN_COST * 3);
9331 format %{ "smull $dst, $src1, $src2" %}
9332
9333 ins_encode %{
9334 __ smull(as_Register($dst$$reg),
9335 as_Register($src1$$reg),
9336 as_Register($src2$$reg));
9337 %}
9338
9339 ins_pipe(imul_reg_reg);
9340 %}
9341
9342 // Long Multiply
9343
9344 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9345 match(Set dst (MulL src1 src2));
9346
9347 ins_cost(INSN_COST * 5);
9348 format %{ "mul $dst, $src1, $src2" %}
9349
9350 ins_encode %{
9351 __ mul(as_Register($dst$$reg),
9352 as_Register($src1$$reg),
9353 as_Register($src2$$reg));
9354 %}
9355
9356 ins_pipe(lmul_reg_reg);
9357 %}
9358
9359 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9360 %{
9361 match(Set dst (MulHiL src1 src2));
9362
9363 ins_cost(INSN_COST * 7);
9364 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9365
9366 ins_encode %{
9367 __ smulh(as_Register($dst$$reg),
9368 as_Register($src1$$reg),
9369 as_Register($src2$$reg));
9370 %}
9371
9372 ins_pipe(lmul_reg_reg);
9373 %}
9374
9375 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9376 %{
9377 match(Set dst (UMulHiL src1 src2));
9378
9379 ins_cost(INSN_COST * 7);
9380 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9381
9382 ins_encode %{
9383 __ umulh(as_Register($dst$$reg),
9384 as_Register($src1$$reg),
9385 as_Register($src2$$reg));
9386 %}
9387
9388 ins_pipe(lmul_reg_reg);
9389 %}
9390
9391 // Combined Integer Multiply & Add/Sub
9392
9393 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9394 match(Set dst (AddI src3 (MulI src1 src2)));
9395
9396 ins_cost(INSN_COST * 3);
9397 format %{ "madd $dst, $src1, $src2, $src3" %}
9398
9399 ins_encode %{
9400 __ maddw(as_Register($dst$$reg),
9401 as_Register($src1$$reg),
9402 as_Register($src2$$reg),
9403 as_Register($src3$$reg));
9404 %}
9405
9406 ins_pipe(imac_reg_reg);
9407 %}
9408
9409 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9410 match(Set dst (SubI src3 (MulI src1 src2)));
9411
9412 ins_cost(INSN_COST * 3);
9413 format %{ "msub $dst, $src1, $src2, $src3" %}
9414
9415 ins_encode %{
9416 __ msubw(as_Register($dst$$reg),
9417 as_Register($src1$$reg),
9418 as_Register($src2$$reg),
9419 as_Register($src3$$reg));
9420 %}
9421
9422 ins_pipe(imac_reg_reg);
9423 %}
9424
9425 // Combined Integer Multiply & Neg
9426
9427 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9428 match(Set dst (MulI (SubI zero src1) src2));
9429
9430 ins_cost(INSN_COST * 3);
9431 format %{ "mneg $dst, $src1, $src2" %}
9432
9433 ins_encode %{
9434 __ mnegw(as_Register($dst$$reg),
9435 as_Register($src1$$reg),
9436 as_Register($src2$$reg));
9437 %}
9438
9439 ins_pipe(imac_reg_reg);
9440 %}
9441
9442 // Combined Long Multiply & Add/Sub
9443
9444 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9445 match(Set dst (AddL src3 (MulL src1 src2)));
9446
9447 ins_cost(INSN_COST * 5);
9448 format %{ "madd $dst, $src1, $src2, $src3" %}
9449
9450 ins_encode %{
9451 __ madd(as_Register($dst$$reg),
9452 as_Register($src1$$reg),
9453 as_Register($src2$$reg),
9454 as_Register($src3$$reg));
9455 %}
9456
9457 ins_pipe(lmac_reg_reg);
9458 %}
9459
9460 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9461 match(Set dst (SubL src3 (MulL src1 src2)));
9462
9463 ins_cost(INSN_COST * 5);
9464 format %{ "msub $dst, $src1, $src2, $src3" %}
9465
9466 ins_encode %{
9467 __ msub(as_Register($dst$$reg),
9468 as_Register($src1$$reg),
9469 as_Register($src2$$reg),
9470 as_Register($src3$$reg));
9471 %}
9472
9473 ins_pipe(lmac_reg_reg);
9474 %}
9475
9476 // Combined Long Multiply & Neg
9477
9478 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9479 match(Set dst (MulL (SubL zero src1) src2));
9480
9481 ins_cost(INSN_COST * 5);
9482 format %{ "mneg $dst, $src1, $src2" %}
9483
9484 ins_encode %{
9485 __ mneg(as_Register($dst$$reg),
9486 as_Register($src1$$reg),
9487 as_Register($src2$$reg));
9488 %}
9489
9490 ins_pipe(lmac_reg_reg);
9491 %}
9492
9493 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9494
9495 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9496 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9497
9498 ins_cost(INSN_COST * 3);
9499 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9500
9501 ins_encode %{
9502 __ smaddl(as_Register($dst$$reg),
9503 as_Register($src1$$reg),
9504 as_Register($src2$$reg),
9505 as_Register($src3$$reg));
9506 %}
9507
9508 ins_pipe(imac_reg_reg);
9509 %}
9510
9511 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9512 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9513
9514 ins_cost(INSN_COST * 3);
9515 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9516
9517 ins_encode %{
9518 __ smsubl(as_Register($dst$$reg),
9519 as_Register($src1$$reg),
9520 as_Register($src2$$reg),
9521 as_Register($src3$$reg));
9522 %}
9523
9524 ins_pipe(imac_reg_reg);
9525 %}
9526
9527 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9528 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9529
9530 ins_cost(INSN_COST * 3);
9531 format %{ "smnegl $dst, $src1, $src2" %}
9532
9533 ins_encode %{
9534 __ smnegl(as_Register($dst$$reg),
9535 as_Register($src1$$reg),
9536 as_Register($src2$$reg));
9537 %}
9538
9539 ins_pipe(imac_reg_reg);
9540 %}
9541
9542 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9543
9544 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9545 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9546
9547 ins_cost(INSN_COST * 5);
9548 format %{ "mulw rscratch1, $src1, $src2\n\t"
9549 "maddw $dst, $src3, $src4, rscratch1" %}
9550
9551 ins_encode %{
9552 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9553 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9554
9555 ins_pipe(imac_reg_reg);
9556 %}
9557
9558 // Integer Divide
9559
9560 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9561 match(Set dst (DivI src1 src2));
9562
9563 ins_cost(INSN_COST * 19);
9564 format %{ "sdivw $dst, $src1, $src2" %}
9565
9566 ins_encode(aarch64_enc_divw(dst, src1, src2));
9567 ins_pipe(idiv_reg_reg);
9568 %}
9569
9570 // Long Divide
9571
9572 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9573 match(Set dst (DivL src1 src2));
9574
9575 ins_cost(INSN_COST * 35);
9576 format %{ "sdiv $dst, $src1, $src2" %}
9577
9578 ins_encode(aarch64_enc_div(dst, src1, src2));
9579 ins_pipe(ldiv_reg_reg);
9580 %}
9581
9582 // Integer Remainder
9583
9584 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9585 match(Set dst (ModI src1 src2));
9586
9587 ins_cost(INSN_COST * 22);
9588 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9589 "msubw $dst, rscratch1, $src2, $src1" %}
9590
9591 ins_encode(aarch64_enc_modw(dst, src1, src2));
9592 ins_pipe(idiv_reg_reg);
9593 %}
9594
9595 // Long Remainder
9596
9597 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9598 match(Set dst (ModL src1 src2));
9599
9600 ins_cost(INSN_COST * 38);
9601 format %{ "sdiv rscratch1, $src1, $src2\n"
9602 "msub $dst, rscratch1, $src2, $src1" %}
9603
9604 ins_encode(aarch64_enc_mod(dst, src1, src2));
9605 ins_pipe(ldiv_reg_reg);
9606 %}
9607
9608 // Unsigned Integer Divide
9609
9610 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9611 match(Set dst (UDivI src1 src2));
9612
9613 ins_cost(INSN_COST * 19);
9614 format %{ "udivw $dst, $src1, $src2" %}
9615
9616 ins_encode %{
9617 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9618 %}
9619
9620 ins_pipe(idiv_reg_reg);
9621 %}
9622
9623 // Unsigned Long Divide
9624
9625 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9626 match(Set dst (UDivL src1 src2));
9627
9628 ins_cost(INSN_COST * 35);
9629 format %{ "udiv $dst, $src1, $src2" %}
9630
9631 ins_encode %{
9632 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9633 %}
9634
9635 ins_pipe(ldiv_reg_reg);
9636 %}
9637
9638 // Unsigned Integer Remainder
9639
9640 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9641 match(Set dst (UModI src1 src2));
9642
9643 ins_cost(INSN_COST * 22);
9644 format %{ "udivw rscratch1, $src1, $src2\n\t"
9645 "msubw $dst, rscratch1, $src2, $src1" %}
9646
9647 ins_encode %{
9648 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9649 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9650 %}
9651
9652 ins_pipe(idiv_reg_reg);
9653 %}
9654
9655 // Unsigned Long Remainder
9656
9657 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9658 match(Set dst (UModL src1 src2));
9659
9660 ins_cost(INSN_COST * 38);
9661 format %{ "udiv rscratch1, $src1, $src2\n"
9662 "msub $dst, rscratch1, $src2, $src1" %}
9663
9664 ins_encode %{
9665 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9666 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9667 %}
9668
9669 ins_pipe(ldiv_reg_reg);
9670 %}
9671
9672 // Integer Shifts
9673
9674 // Shift Left Register
9675 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9676 match(Set dst (LShiftI src1 src2));
9677
9678 ins_cost(INSN_COST * 2);
9679 format %{ "lslvw $dst, $src1, $src2" %}
9680
9681 ins_encode %{
9682 __ lslvw(as_Register($dst$$reg),
9683 as_Register($src1$$reg),
9684 as_Register($src2$$reg));
9685 %}
9686
9687 ins_pipe(ialu_reg_reg_vshift);
9688 %}
9689
9690 // Shift Left Immediate
9691 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9692 match(Set dst (LShiftI src1 src2));
9693
9694 ins_cost(INSN_COST);
9695 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9696
9697 ins_encode %{
9698 __ lslw(as_Register($dst$$reg),
9699 as_Register($src1$$reg),
9700 $src2$$constant & 0x1f);
9701 %}
9702
9703 ins_pipe(ialu_reg_shift);
9704 %}
9705
9706 // Shift Right Logical Register
9707 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9708 match(Set dst (URShiftI src1 src2));
9709
9710 ins_cost(INSN_COST * 2);
9711 format %{ "lsrvw $dst, $src1, $src2" %}
9712
9713 ins_encode %{
9714 __ lsrvw(as_Register($dst$$reg),
9715 as_Register($src1$$reg),
9716 as_Register($src2$$reg));
9717 %}
9718
9719 ins_pipe(ialu_reg_reg_vshift);
9720 %}
9721
9722 // Shift Right Logical Immediate
9723 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9724 match(Set dst (URShiftI src1 src2));
9725
9726 ins_cost(INSN_COST);
9727 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9728
9729 ins_encode %{
9730 __ lsrw(as_Register($dst$$reg),
9731 as_Register($src1$$reg),
9732 $src2$$constant & 0x1f);
9733 %}
9734
9735 ins_pipe(ialu_reg_shift);
9736 %}
9737
9738 // Shift Right Arithmetic Register
9739 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9740 match(Set dst (RShiftI src1 src2));
9741
9742 ins_cost(INSN_COST * 2);
9743 format %{ "asrvw $dst, $src1, $src2" %}
9744
9745 ins_encode %{
9746 __ asrvw(as_Register($dst$$reg),
9747 as_Register($src1$$reg),
9748 as_Register($src2$$reg));
9749 %}
9750
9751 ins_pipe(ialu_reg_reg_vshift);
9752 %}
9753
9754 // Shift Right Arithmetic Immediate
9755 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9756 match(Set dst (RShiftI src1 src2));
9757
9758 ins_cost(INSN_COST);
9759 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9760
9761 ins_encode %{
9762 __ asrw(as_Register($dst$$reg),
9763 as_Register($src1$$reg),
9764 $src2$$constant & 0x1f);
9765 %}
9766
9767 ins_pipe(ialu_reg_shift);
9768 %}
9769
9770 // Combined Int Mask and Right Shift (using UBFM)
9771 // TODO
9772
9773 // Long Shifts
9774
9775 // Shift Left Register
9776 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9777 match(Set dst (LShiftL src1 src2));
9778
9779 ins_cost(INSN_COST * 2);
9780 format %{ "lslv $dst, $src1, $src2" %}
9781
9782 ins_encode %{
9783 __ lslv(as_Register($dst$$reg),
9784 as_Register($src1$$reg),
9785 as_Register($src2$$reg));
9786 %}
9787
9788 ins_pipe(ialu_reg_reg_vshift);
9789 %}
9790
9791 // Shift Left Immediate
9792 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9793 match(Set dst (LShiftL src1 src2));
9794
9795 ins_cost(INSN_COST);
9796 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9797
9798 ins_encode %{
9799 __ lsl(as_Register($dst$$reg),
9800 as_Register($src1$$reg),
9801 $src2$$constant & 0x3f);
9802 %}
9803
9804 ins_pipe(ialu_reg_shift);
9805 %}
9806
9807 // Shift Right Logical Register
9808 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9809 match(Set dst (URShiftL src1 src2));
9810
9811 ins_cost(INSN_COST * 2);
9812 format %{ "lsrv $dst, $src1, $src2" %}
9813
9814 ins_encode %{
9815 __ lsrv(as_Register($dst$$reg),
9816 as_Register($src1$$reg),
9817 as_Register($src2$$reg));
9818 %}
9819
9820 ins_pipe(ialu_reg_reg_vshift);
9821 %}
9822
9823 // Shift Right Logical Immediate
9824 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9825 match(Set dst (URShiftL src1 src2));
9826
9827 ins_cost(INSN_COST);
9828 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9829
9830 ins_encode %{
9831 __ lsr(as_Register($dst$$reg),
9832 as_Register($src1$$reg),
9833 $src2$$constant & 0x3f);
9834 %}
9835
9836 ins_pipe(ialu_reg_shift);
9837 %}
9838
9839 // A special-case pattern for card table stores.
9840 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9841 match(Set dst (URShiftL (CastP2X src1) src2));
9842
9843 ins_cost(INSN_COST);
9844 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9845
9846 ins_encode %{
9847 __ lsr(as_Register($dst$$reg),
9848 as_Register($src1$$reg),
9849 $src2$$constant & 0x3f);
9850 %}
9851
9852 ins_pipe(ialu_reg_shift);
9853 %}
9854
9855 // Shift Right Arithmetic Register
9856 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9857 match(Set dst (RShiftL src1 src2));
9858
9859 ins_cost(INSN_COST * 2);
9860 format %{ "asrv $dst, $src1, $src2" %}
9861
9862 ins_encode %{
9863 __ asrv(as_Register($dst$$reg),
9864 as_Register($src1$$reg),
9865 as_Register($src2$$reg));
9866 %}
9867
9868 ins_pipe(ialu_reg_reg_vshift);
9869 %}
9870
9871 // Shift Right Arithmetic Immediate
9872 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9873 match(Set dst (RShiftL src1 src2));
9874
9875 ins_cost(INSN_COST);
9876 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9877
9878 ins_encode %{
9879 __ asr(as_Register($dst$$reg),
9880 as_Register($src1$$reg),
9881 $src2$$constant & 0x3f);
9882 %}
9883
9884 ins_pipe(ialu_reg_shift);
9885 %}
9886
9887 // BEGIN This section of the file is automatically generated. Do not edit --------------
9888 // This section is generated from aarch64_ad.m4
9889
9890 // This pattern is automatically generated from aarch64_ad.m4
9891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9892 instruct regL_not_reg(iRegLNoSp dst,
9893 iRegL src1, immL_M1 m1,
9894 rFlagsReg cr) %{
9895 match(Set dst (XorL src1 m1));
9896 ins_cost(INSN_COST);
9897 format %{ "eon $dst, $src1, zr" %}
9898
9899 ins_encode %{
9900 __ eon(as_Register($dst$$reg),
9901 as_Register($src1$$reg),
9902 zr,
9903 Assembler::LSL, 0);
9904 %}
9905
9906 ins_pipe(ialu_reg);
9907 %}
9908
9909 // This pattern is automatically generated from aarch64_ad.m4
9910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9911 instruct regI_not_reg(iRegINoSp dst,
9912 iRegIorL2I src1, immI_M1 m1,
9913 rFlagsReg cr) %{
9914 match(Set dst (XorI src1 m1));
9915 ins_cost(INSN_COST);
9916 format %{ "eonw $dst, $src1, zr" %}
9917
9918 ins_encode %{
9919 __ eonw(as_Register($dst$$reg),
9920 as_Register($src1$$reg),
9921 zr,
9922 Assembler::LSL, 0);
9923 %}
9924
9925 ins_pipe(ialu_reg);
9926 %}
9927
9928 // This pattern is automatically generated from aarch64_ad.m4
9929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9930 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9931 immI0 zero, iRegIorL2I src1, immI src2) %{
9932 match(Set dst (SubI zero (URShiftI src1 src2)));
9933
9934 ins_cost(1.9 * INSN_COST);
9935 format %{ "negw $dst, $src1, LSR $src2" %}
9936
9937 ins_encode %{
9938 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9939 Assembler::LSR, $src2$$constant & 0x1f);
9940 %}
9941
9942 ins_pipe(ialu_reg_shift);
9943 %}
9944
9945 // This pattern is automatically generated from aarch64_ad.m4
9946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9947 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9948 immI0 zero, iRegIorL2I src1, immI src2) %{
9949 match(Set dst (SubI zero (RShiftI src1 src2)));
9950
9951 ins_cost(1.9 * INSN_COST);
9952 format %{ "negw $dst, $src1, ASR $src2" %}
9953
9954 ins_encode %{
9955 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9956 Assembler::ASR, $src2$$constant & 0x1f);
9957 %}
9958
9959 ins_pipe(ialu_reg_shift);
9960 %}
9961
9962 // This pattern is automatically generated from aarch64_ad.m4
9963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9964 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9965 immI0 zero, iRegIorL2I src1, immI src2) %{
9966 match(Set dst (SubI zero (LShiftI src1 src2)));
9967
9968 ins_cost(1.9 * INSN_COST);
9969 format %{ "negw $dst, $src1, LSL $src2" %}
9970
9971 ins_encode %{
9972 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9973 Assembler::LSL, $src2$$constant & 0x1f);
9974 %}
9975
9976 ins_pipe(ialu_reg_shift);
9977 %}
9978
9979 // This pattern is automatically generated from aarch64_ad.m4
9980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9981 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9982 immL0 zero, iRegL src1, immI src2) %{
9983 match(Set dst (SubL zero (URShiftL src1 src2)));
9984
9985 ins_cost(1.9 * INSN_COST);
9986 format %{ "neg $dst, $src1, LSR $src2" %}
9987
9988 ins_encode %{
9989 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9990 Assembler::LSR, $src2$$constant & 0x3f);
9991 %}
9992
9993 ins_pipe(ialu_reg_shift);
9994 %}
9995
9996 // This pattern is automatically generated from aarch64_ad.m4
9997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9998 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9999 immL0 zero, iRegL src1, immI src2) %{
10000 match(Set dst (SubL zero (RShiftL src1 src2)));
10001
10002 ins_cost(1.9 * INSN_COST);
10003 format %{ "neg $dst, $src1, ASR $src2" %}
10004
10005 ins_encode %{
10006 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10007 Assembler::ASR, $src2$$constant & 0x3f);
10008 %}
10009
10010 ins_pipe(ialu_reg_shift);
10011 %}
10012
10013 // This pattern is automatically generated from aarch64_ad.m4
10014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10015 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10016 immL0 zero, iRegL src1, immI src2) %{
10017 match(Set dst (SubL zero (LShiftL src1 src2)));
10018
10019 ins_cost(1.9 * INSN_COST);
10020 format %{ "neg $dst, $src1, LSL $src2" %}
10021
10022 ins_encode %{
10023 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10024 Assembler::LSL, $src2$$constant & 0x3f);
10025 %}
10026
10027 ins_pipe(ialu_reg_shift);
10028 %}
10029
10030 // This pattern is automatically generated from aarch64_ad.m4
10031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10032 instruct AndI_reg_not_reg(iRegINoSp dst,
10033 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10034 match(Set dst (AndI src1 (XorI src2 m1)));
10035 ins_cost(INSN_COST);
10036 format %{ "bicw $dst, $src1, $src2" %}
10037
10038 ins_encode %{
10039 __ bicw(as_Register($dst$$reg),
10040 as_Register($src1$$reg),
10041 as_Register($src2$$reg),
10042 Assembler::LSL, 0);
10043 %}
10044
10045 ins_pipe(ialu_reg_reg);
10046 %}
10047
10048 // This pattern is automatically generated from aarch64_ad.m4
10049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10050 instruct AndL_reg_not_reg(iRegLNoSp dst,
10051 iRegL src1, iRegL src2, immL_M1 m1) %{
10052 match(Set dst (AndL src1 (XorL src2 m1)));
10053 ins_cost(INSN_COST);
10054 format %{ "bic $dst, $src1, $src2" %}
10055
10056 ins_encode %{
10057 __ bic(as_Register($dst$$reg),
10058 as_Register($src1$$reg),
10059 as_Register($src2$$reg),
10060 Assembler::LSL, 0);
10061 %}
10062
10063 ins_pipe(ialu_reg_reg);
10064 %}
10065
10066 // This pattern is automatically generated from aarch64_ad.m4
10067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10068 instruct OrI_reg_not_reg(iRegINoSp dst,
10069 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10070 match(Set dst (OrI src1 (XorI src2 m1)));
10071 ins_cost(INSN_COST);
10072 format %{ "ornw $dst, $src1, $src2" %}
10073
10074 ins_encode %{
10075 __ ornw(as_Register($dst$$reg),
10076 as_Register($src1$$reg),
10077 as_Register($src2$$reg),
10078 Assembler::LSL, 0);
10079 %}
10080
10081 ins_pipe(ialu_reg_reg);
10082 %}
10083
10084 // This pattern is automatically generated from aarch64_ad.m4
10085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10086 instruct OrL_reg_not_reg(iRegLNoSp dst,
10087 iRegL src1, iRegL src2, immL_M1 m1) %{
10088 match(Set dst (OrL src1 (XorL src2 m1)));
10089 ins_cost(INSN_COST);
10090 format %{ "orn $dst, $src1, $src2" %}
10091
10092 ins_encode %{
10093 __ orn(as_Register($dst$$reg),
10094 as_Register($src1$$reg),
10095 as_Register($src2$$reg),
10096 Assembler::LSL, 0);
10097 %}
10098
10099 ins_pipe(ialu_reg_reg);
10100 %}
10101
10102 // This pattern is automatically generated from aarch64_ad.m4
10103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10104 instruct XorI_reg_not_reg(iRegINoSp dst,
10105 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10106 match(Set dst (XorI m1 (XorI src2 src1)));
10107 ins_cost(INSN_COST);
10108 format %{ "eonw $dst, $src1, $src2" %}
10109
10110 ins_encode %{
10111 __ eonw(as_Register($dst$$reg),
10112 as_Register($src1$$reg),
10113 as_Register($src2$$reg),
10114 Assembler::LSL, 0);
10115 %}
10116
10117 ins_pipe(ialu_reg_reg);
10118 %}
10119
10120 // This pattern is automatically generated from aarch64_ad.m4
10121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10122 instruct XorL_reg_not_reg(iRegLNoSp dst,
10123 iRegL src1, iRegL src2, immL_M1 m1) %{
10124 match(Set dst (XorL m1 (XorL src2 src1)));
10125 ins_cost(INSN_COST);
10126 format %{ "eon $dst, $src1, $src2" %}
10127
10128 ins_encode %{
10129 __ eon(as_Register($dst$$reg),
10130 as_Register($src1$$reg),
10131 as_Register($src2$$reg),
10132 Assembler::LSL, 0);
10133 %}
10134
10135 ins_pipe(ialu_reg_reg);
10136 %}
10137
10138 // This pattern is automatically generated from aarch64_ad.m4
10139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10140 // val & (-1 ^ (val >>> shift)) ==> bicw
10141 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10142 iRegIorL2I src1, iRegIorL2I src2,
10143 immI src3, immI_M1 src4) %{
10144 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10145 ins_cost(1.9 * INSN_COST);
10146 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10147
10148 ins_encode %{
10149 __ bicw(as_Register($dst$$reg),
10150 as_Register($src1$$reg),
10151 as_Register($src2$$reg),
10152 Assembler::LSR,
10153 $src3$$constant & 0x1f);
10154 %}
10155
10156 ins_pipe(ialu_reg_reg_shift);
10157 %}
10158
10159 // This pattern is automatically generated from aarch64_ad.m4
10160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10161 // val & (-1 ^ (val >>> shift)) ==> bic
10162 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10163 iRegL src1, iRegL src2,
10164 immI src3, immL_M1 src4) %{
10165 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10166 ins_cost(1.9 * INSN_COST);
10167 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10168
10169 ins_encode %{
10170 __ bic(as_Register($dst$$reg),
10171 as_Register($src1$$reg),
10172 as_Register($src2$$reg),
10173 Assembler::LSR,
10174 $src3$$constant & 0x3f);
10175 %}
10176
10177 ins_pipe(ialu_reg_reg_shift);
10178 %}
10179
10180 // This pattern is automatically generated from aarch64_ad.m4
10181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10182 // val & (-1 ^ (val >> shift)) ==> bicw
10183 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10184 iRegIorL2I src1, iRegIorL2I src2,
10185 immI src3, immI_M1 src4) %{
10186 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10187 ins_cost(1.9 * INSN_COST);
10188 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10189
10190 ins_encode %{
10191 __ bicw(as_Register($dst$$reg),
10192 as_Register($src1$$reg),
10193 as_Register($src2$$reg),
10194 Assembler::ASR,
10195 $src3$$constant & 0x1f);
10196 %}
10197
10198 ins_pipe(ialu_reg_reg_shift);
10199 %}
10200
10201 // This pattern is automatically generated from aarch64_ad.m4
10202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10203 // val & (-1 ^ (val >> shift)) ==> bic
10204 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10205 iRegL src1, iRegL src2,
10206 immI src3, immL_M1 src4) %{
10207 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10208 ins_cost(1.9 * INSN_COST);
10209 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10210
10211 ins_encode %{
10212 __ bic(as_Register($dst$$reg),
10213 as_Register($src1$$reg),
10214 as_Register($src2$$reg),
10215 Assembler::ASR,
10216 $src3$$constant & 0x3f);
10217 %}
10218
10219 ins_pipe(ialu_reg_reg_shift);
10220 %}
10221
10222 // This pattern is automatically generated from aarch64_ad.m4
10223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10224 // val & (-1 ^ (val ror shift)) ==> bicw
10225 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10226 iRegIorL2I src1, iRegIorL2I src2,
10227 immI src3, immI_M1 src4) %{
10228 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10229 ins_cost(1.9 * INSN_COST);
10230 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10231
10232 ins_encode %{
10233 __ bicw(as_Register($dst$$reg),
10234 as_Register($src1$$reg),
10235 as_Register($src2$$reg),
10236 Assembler::ROR,
10237 $src3$$constant & 0x1f);
10238 %}
10239
10240 ins_pipe(ialu_reg_reg_shift);
10241 %}
10242
10243 // This pattern is automatically generated from aarch64_ad.m4
10244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10245 // val & (-1 ^ (val ror shift)) ==> bic
10246 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10247 iRegL src1, iRegL src2,
10248 immI src3, immL_M1 src4) %{
10249 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10250 ins_cost(1.9 * INSN_COST);
10251 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10252
10253 ins_encode %{
10254 __ bic(as_Register($dst$$reg),
10255 as_Register($src1$$reg),
10256 as_Register($src2$$reg),
10257 Assembler::ROR,
10258 $src3$$constant & 0x3f);
10259 %}
10260
10261 ins_pipe(ialu_reg_reg_shift);
10262 %}
10263
10264 // This pattern is automatically generated from aarch64_ad.m4
10265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10266 // val & (-1 ^ (val << shift)) ==> bicw
10267 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10268 iRegIorL2I src1, iRegIorL2I src2,
10269 immI src3, immI_M1 src4) %{
10270 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10271 ins_cost(1.9 * INSN_COST);
10272 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10273
10274 ins_encode %{
10275 __ bicw(as_Register($dst$$reg),
10276 as_Register($src1$$reg),
10277 as_Register($src2$$reg),
10278 Assembler::LSL,
10279 $src3$$constant & 0x1f);
10280 %}
10281
10282 ins_pipe(ialu_reg_reg_shift);
10283 %}
10284
10285 // This pattern is automatically generated from aarch64_ad.m4
10286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10287 // val & (-1 ^ (val << shift)) ==> bic
10288 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10289 iRegL src1, iRegL src2,
10290 immI src3, immL_M1 src4) %{
10291 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10292 ins_cost(1.9 * INSN_COST);
10293 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10294
10295 ins_encode %{
10296 __ bic(as_Register($dst$$reg),
10297 as_Register($src1$$reg),
10298 as_Register($src2$$reg),
10299 Assembler::LSL,
10300 $src3$$constant & 0x3f);
10301 %}
10302
10303 ins_pipe(ialu_reg_reg_shift);
10304 %}
10305
10306 // This pattern is automatically generated from aarch64_ad.m4
10307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10308 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10309 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10310 iRegIorL2I src1, iRegIorL2I src2,
10311 immI src3, immI_M1 src4) %{
10312 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10313 ins_cost(1.9 * INSN_COST);
10314 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10315
10316 ins_encode %{
10317 __ eonw(as_Register($dst$$reg),
10318 as_Register($src1$$reg),
10319 as_Register($src2$$reg),
10320 Assembler::LSR,
10321 $src3$$constant & 0x1f);
10322 %}
10323
10324 ins_pipe(ialu_reg_reg_shift);
10325 %}
10326
10327 // This pattern is automatically generated from aarch64_ad.m4
10328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10329 // val ^ (-1 ^ (val >>> shift)) ==> eon
10330 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10331 iRegL src1, iRegL src2,
10332 immI src3, immL_M1 src4) %{
10333 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10334 ins_cost(1.9 * INSN_COST);
10335 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10336
10337 ins_encode %{
10338 __ eon(as_Register($dst$$reg),
10339 as_Register($src1$$reg),
10340 as_Register($src2$$reg),
10341 Assembler::LSR,
10342 $src3$$constant & 0x3f);
10343 %}
10344
10345 ins_pipe(ialu_reg_reg_shift);
10346 %}
10347
10348 // This pattern is automatically generated from aarch64_ad.m4
10349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10350 // val ^ (-1 ^ (val >> shift)) ==> eonw
10351 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10352 iRegIorL2I src1, iRegIorL2I src2,
10353 immI src3, immI_M1 src4) %{
10354 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10355 ins_cost(1.9 * INSN_COST);
10356 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10357
10358 ins_encode %{
10359 __ eonw(as_Register($dst$$reg),
10360 as_Register($src1$$reg),
10361 as_Register($src2$$reg),
10362 Assembler::ASR,
10363 $src3$$constant & 0x1f);
10364 %}
10365
10366 ins_pipe(ialu_reg_reg_shift);
10367 %}
10368
10369 // This pattern is automatically generated from aarch64_ad.m4
10370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10371 // val ^ (-1 ^ (val >> shift)) ==> eon
10372 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10373 iRegL src1, iRegL src2,
10374 immI src3, immL_M1 src4) %{
10375 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10376 ins_cost(1.9 * INSN_COST);
10377 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10378
10379 ins_encode %{
10380 __ eon(as_Register($dst$$reg),
10381 as_Register($src1$$reg),
10382 as_Register($src2$$reg),
10383 Assembler::ASR,
10384 $src3$$constant & 0x3f);
10385 %}
10386
10387 ins_pipe(ialu_reg_reg_shift);
10388 %}
10389
10390 // This pattern is automatically generated from aarch64_ad.m4
10391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10392 // val ^ (-1 ^ (val ror shift)) ==> eonw
10393 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10394 iRegIorL2I src1, iRegIorL2I src2,
10395 immI src3, immI_M1 src4) %{
10396 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10397 ins_cost(1.9 * INSN_COST);
10398 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10399
10400 ins_encode %{
10401 __ eonw(as_Register($dst$$reg),
10402 as_Register($src1$$reg),
10403 as_Register($src2$$reg),
10404 Assembler::ROR,
10405 $src3$$constant & 0x1f);
10406 %}
10407
10408 ins_pipe(ialu_reg_reg_shift);
10409 %}
10410
10411 // This pattern is automatically generated from aarch64_ad.m4
10412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10413 // val ^ (-1 ^ (val ror shift)) ==> eon
10414 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10415 iRegL src1, iRegL src2,
10416 immI src3, immL_M1 src4) %{
10417 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10418 ins_cost(1.9 * INSN_COST);
10419 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10420
10421 ins_encode %{
10422 __ eon(as_Register($dst$$reg),
10423 as_Register($src1$$reg),
10424 as_Register($src2$$reg),
10425 Assembler::ROR,
10426 $src3$$constant & 0x3f);
10427 %}
10428
10429 ins_pipe(ialu_reg_reg_shift);
10430 %}
10431
10432 // This pattern is automatically generated from aarch64_ad.m4
10433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10434 // val ^ (-1 ^ (val << shift)) ==> eonw
10435 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10436 iRegIorL2I src1, iRegIorL2I src2,
10437 immI src3, immI_M1 src4) %{
10438 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10439 ins_cost(1.9 * INSN_COST);
10440 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10441
10442 ins_encode %{
10443 __ eonw(as_Register($dst$$reg),
10444 as_Register($src1$$reg),
10445 as_Register($src2$$reg),
10446 Assembler::LSL,
10447 $src3$$constant & 0x1f);
10448 %}
10449
10450 ins_pipe(ialu_reg_reg_shift);
10451 %}
10452
10453 // This pattern is automatically generated from aarch64_ad.m4
10454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10455 // val ^ (-1 ^ (val << shift)) ==> eon
10456 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10457 iRegL src1, iRegL src2,
10458 immI src3, immL_M1 src4) %{
10459 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10460 ins_cost(1.9 * INSN_COST);
10461 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10462
10463 ins_encode %{
10464 __ eon(as_Register($dst$$reg),
10465 as_Register($src1$$reg),
10466 as_Register($src2$$reg),
10467 Assembler::LSL,
10468 $src3$$constant & 0x3f);
10469 %}
10470
10471 ins_pipe(ialu_reg_reg_shift);
10472 %}
10473
10474 // This pattern is automatically generated from aarch64_ad.m4
10475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10476 // val | (-1 ^ (val >>> shift)) ==> ornw
10477 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10478 iRegIorL2I src1, iRegIorL2I src2,
10479 immI src3, immI_M1 src4) %{
10480 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10481 ins_cost(1.9 * INSN_COST);
10482 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10483
10484 ins_encode %{
10485 __ ornw(as_Register($dst$$reg),
10486 as_Register($src1$$reg),
10487 as_Register($src2$$reg),
10488 Assembler::LSR,
10489 $src3$$constant & 0x1f);
10490 %}
10491
10492 ins_pipe(ialu_reg_reg_shift);
10493 %}
10494
10495 // This pattern is automatically generated from aarch64_ad.m4
10496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10497 // val | (-1 ^ (val >>> shift)) ==> orn
10498 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10499 iRegL src1, iRegL src2,
10500 immI src3, immL_M1 src4) %{
10501 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10502 ins_cost(1.9 * INSN_COST);
10503 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10504
10505 ins_encode %{
10506 __ orn(as_Register($dst$$reg),
10507 as_Register($src1$$reg),
10508 as_Register($src2$$reg),
10509 Assembler::LSR,
10510 $src3$$constant & 0x3f);
10511 %}
10512
10513 ins_pipe(ialu_reg_reg_shift);
10514 %}
10515
10516 // This pattern is automatically generated from aarch64_ad.m4
10517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10518 // val | (-1 ^ (val >> shift)) ==> ornw
10519 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10520 iRegIorL2I src1, iRegIorL2I src2,
10521 immI src3, immI_M1 src4) %{
10522 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10523 ins_cost(1.9 * INSN_COST);
10524 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10525
10526 ins_encode %{
10527 __ ornw(as_Register($dst$$reg),
10528 as_Register($src1$$reg),
10529 as_Register($src2$$reg),
10530 Assembler::ASR,
10531 $src3$$constant & 0x1f);
10532 %}
10533
10534 ins_pipe(ialu_reg_reg_shift);
10535 %}
10536
10537 // This pattern is automatically generated from aarch64_ad.m4
10538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10539 // val | (-1 ^ (val >> shift)) ==> orn
10540 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10541 iRegL src1, iRegL src2,
10542 immI src3, immL_M1 src4) %{
10543 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10544 ins_cost(1.9 * INSN_COST);
10545 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10546
10547 ins_encode %{
10548 __ orn(as_Register($dst$$reg),
10549 as_Register($src1$$reg),
10550 as_Register($src2$$reg),
10551 Assembler::ASR,
10552 $src3$$constant & 0x3f);
10553 %}
10554
10555 ins_pipe(ialu_reg_reg_shift);
10556 %}
10557
10558 // This pattern is automatically generated from aarch64_ad.m4
10559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10560 // val | (-1 ^ (val ror shift)) ==> ornw
10561 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10562 iRegIorL2I src1, iRegIorL2I src2,
10563 immI src3, immI_M1 src4) %{
10564 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10565 ins_cost(1.9 * INSN_COST);
10566 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10567
10568 ins_encode %{
10569 __ ornw(as_Register($dst$$reg),
10570 as_Register($src1$$reg),
10571 as_Register($src2$$reg),
10572 Assembler::ROR,
10573 $src3$$constant & 0x1f);
10574 %}
10575
10576 ins_pipe(ialu_reg_reg_shift);
10577 %}
10578
10579 // This pattern is automatically generated from aarch64_ad.m4
10580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10581 // val | (-1 ^ (val ror shift)) ==> orn
10582 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10583 iRegL src1, iRegL src2,
10584 immI src3, immL_M1 src4) %{
10585 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10586 ins_cost(1.9 * INSN_COST);
10587 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10588
10589 ins_encode %{
10590 __ orn(as_Register($dst$$reg),
10591 as_Register($src1$$reg),
10592 as_Register($src2$$reg),
10593 Assembler::ROR,
10594 $src3$$constant & 0x3f);
10595 %}
10596
10597 ins_pipe(ialu_reg_reg_shift);
10598 %}
10599
10600 // This pattern is automatically generated from aarch64_ad.m4
10601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10602 // val | (-1 ^ (val << shift)) ==> ornw
10603 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10604 iRegIorL2I src1, iRegIorL2I src2,
10605 immI src3, immI_M1 src4) %{
10606 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10607 ins_cost(1.9 * INSN_COST);
10608 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10609
10610 ins_encode %{
10611 __ ornw(as_Register($dst$$reg),
10612 as_Register($src1$$reg),
10613 as_Register($src2$$reg),
10614 Assembler::LSL,
10615 $src3$$constant & 0x1f);
10616 %}
10617
10618 ins_pipe(ialu_reg_reg_shift);
10619 %}
10620
10621 // This pattern is automatically generated from aarch64_ad.m4
10622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10623 // val | (-1 ^ (val << shift)) ==> orn
10624 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10625 iRegL src1, iRegL src2,
10626 immI src3, immL_M1 src4) %{
10627 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10628 ins_cost(1.9 * INSN_COST);
10629 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10630
10631 ins_encode %{
10632 __ orn(as_Register($dst$$reg),
10633 as_Register($src1$$reg),
10634 as_Register($src2$$reg),
10635 Assembler::LSL,
10636 $src3$$constant & 0x3f);
10637 %}
10638
10639 ins_pipe(ialu_reg_reg_shift);
10640 %}
10641
10642 // This pattern is automatically generated from aarch64_ad.m4
10643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10644 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10645 iRegIorL2I src1, iRegIorL2I src2,
10646 immI src3) %{
10647 match(Set dst (AndI src1 (URShiftI src2 src3)));
10648
10649 ins_cost(1.9 * INSN_COST);
10650 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10651
10652 ins_encode %{
10653 __ andw(as_Register($dst$$reg),
10654 as_Register($src1$$reg),
10655 as_Register($src2$$reg),
10656 Assembler::LSR,
10657 $src3$$constant & 0x1f);
10658 %}
10659
10660 ins_pipe(ialu_reg_reg_shift);
10661 %}
10662
10663 // This pattern is automatically generated from aarch64_ad.m4
10664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10665 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10666 iRegL src1, iRegL src2,
10667 immI src3) %{
10668 match(Set dst (AndL src1 (URShiftL src2 src3)));
10669
10670 ins_cost(1.9 * INSN_COST);
10671 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10672
10673 ins_encode %{
10674 __ andr(as_Register($dst$$reg),
10675 as_Register($src1$$reg),
10676 as_Register($src2$$reg),
10677 Assembler::LSR,
10678 $src3$$constant & 0x3f);
10679 %}
10680
10681 ins_pipe(ialu_reg_reg_shift);
10682 %}
10683
10684 // This pattern is automatically generated from aarch64_ad.m4
10685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10686 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10687 iRegIorL2I src1, iRegIorL2I src2,
10688 immI src3) %{
10689 match(Set dst (AndI src1 (RShiftI src2 src3)));
10690
10691 ins_cost(1.9 * INSN_COST);
10692 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10693
10694 ins_encode %{
10695 __ andw(as_Register($dst$$reg),
10696 as_Register($src1$$reg),
10697 as_Register($src2$$reg),
10698 Assembler::ASR,
10699 $src3$$constant & 0x1f);
10700 %}
10701
10702 ins_pipe(ialu_reg_reg_shift);
10703 %}
10704
10705 // This pattern is automatically generated from aarch64_ad.m4
10706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10707 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10708 iRegL src1, iRegL src2,
10709 immI src3) %{
10710 match(Set dst (AndL src1 (RShiftL src2 src3)));
10711
10712 ins_cost(1.9 * INSN_COST);
10713 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10714
10715 ins_encode %{
10716 __ andr(as_Register($dst$$reg),
10717 as_Register($src1$$reg),
10718 as_Register($src2$$reg),
10719 Assembler::ASR,
10720 $src3$$constant & 0x3f);
10721 %}
10722
10723 ins_pipe(ialu_reg_reg_shift);
10724 %}
10725
10726 // This pattern is automatically generated from aarch64_ad.m4
10727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10728 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10729 iRegIorL2I src1, iRegIorL2I src2,
10730 immI src3) %{
10731 match(Set dst (AndI src1 (LShiftI src2 src3)));
10732
10733 ins_cost(1.9 * INSN_COST);
10734 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10735
10736 ins_encode %{
10737 __ andw(as_Register($dst$$reg),
10738 as_Register($src1$$reg),
10739 as_Register($src2$$reg),
10740 Assembler::LSL,
10741 $src3$$constant & 0x1f);
10742 %}
10743
10744 ins_pipe(ialu_reg_reg_shift);
10745 %}
10746
10747 // This pattern is automatically generated from aarch64_ad.m4
10748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10749 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10750 iRegL src1, iRegL src2,
10751 immI src3) %{
10752 match(Set dst (AndL src1 (LShiftL src2 src3)));
10753
10754 ins_cost(1.9 * INSN_COST);
10755 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10756
10757 ins_encode %{
10758 __ andr(as_Register($dst$$reg),
10759 as_Register($src1$$reg),
10760 as_Register($src2$$reg),
10761 Assembler::LSL,
10762 $src3$$constant & 0x3f);
10763 %}
10764
10765 ins_pipe(ialu_reg_reg_shift);
10766 %}
10767
10768 // This pattern is automatically generated from aarch64_ad.m4
10769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10770 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10771 iRegIorL2I src1, iRegIorL2I src2,
10772 immI src3) %{
10773 match(Set dst (AndI src1 (RotateRight src2 src3)));
10774
10775 ins_cost(1.9 * INSN_COST);
10776 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10777
10778 ins_encode %{
10779 __ andw(as_Register($dst$$reg),
10780 as_Register($src1$$reg),
10781 as_Register($src2$$reg),
10782 Assembler::ROR,
10783 $src3$$constant & 0x1f);
10784 %}
10785
10786 ins_pipe(ialu_reg_reg_shift);
10787 %}
10788
10789 // This pattern is automatically generated from aarch64_ad.m4
10790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10791 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10792 iRegL src1, iRegL src2,
10793 immI src3) %{
10794 match(Set dst (AndL src1 (RotateRight src2 src3)));
10795
10796 ins_cost(1.9 * INSN_COST);
10797 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10798
10799 ins_encode %{
10800 __ andr(as_Register($dst$$reg),
10801 as_Register($src1$$reg),
10802 as_Register($src2$$reg),
10803 Assembler::ROR,
10804 $src3$$constant & 0x3f);
10805 %}
10806
10807 ins_pipe(ialu_reg_reg_shift);
10808 %}
10809
10810 // This pattern is automatically generated from aarch64_ad.m4
10811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10812 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10813 iRegIorL2I src1, iRegIorL2I src2,
10814 immI src3) %{
10815 match(Set dst (XorI src1 (URShiftI src2 src3)));
10816
10817 ins_cost(1.9 * INSN_COST);
10818 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10819
10820 ins_encode %{
10821 __ eorw(as_Register($dst$$reg),
10822 as_Register($src1$$reg),
10823 as_Register($src2$$reg),
10824 Assembler::LSR,
10825 $src3$$constant & 0x1f);
10826 %}
10827
10828 ins_pipe(ialu_reg_reg_shift);
10829 %}
10830
10831 // This pattern is automatically generated from aarch64_ad.m4
10832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10833 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10834 iRegL src1, iRegL src2,
10835 immI src3) %{
10836 match(Set dst (XorL src1 (URShiftL src2 src3)));
10837
10838 ins_cost(1.9 * INSN_COST);
10839 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10840
10841 ins_encode %{
10842 __ eor(as_Register($dst$$reg),
10843 as_Register($src1$$reg),
10844 as_Register($src2$$reg),
10845 Assembler::LSR,
10846 $src3$$constant & 0x3f);
10847 %}
10848
10849 ins_pipe(ialu_reg_reg_shift);
10850 %}
10851
10852 // This pattern is automatically generated from aarch64_ad.m4
10853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10854 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10855 iRegIorL2I src1, iRegIorL2I src2,
10856 immI src3) %{
10857 match(Set dst (XorI src1 (RShiftI src2 src3)));
10858
10859 ins_cost(1.9 * INSN_COST);
10860 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10861
10862 ins_encode %{
10863 __ eorw(as_Register($dst$$reg),
10864 as_Register($src1$$reg),
10865 as_Register($src2$$reg),
10866 Assembler::ASR,
10867 $src3$$constant & 0x1f);
10868 %}
10869
10870 ins_pipe(ialu_reg_reg_shift);
10871 %}
10872
10873 // This pattern is automatically generated from aarch64_ad.m4
10874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10875 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10876 iRegL src1, iRegL src2,
10877 immI src3) %{
10878 match(Set dst (XorL src1 (RShiftL src2 src3)));
10879
10880 ins_cost(1.9 * INSN_COST);
10881 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10882
10883 ins_encode %{
10884 __ eor(as_Register($dst$$reg),
10885 as_Register($src1$$reg),
10886 as_Register($src2$$reg),
10887 Assembler::ASR,
10888 $src3$$constant & 0x3f);
10889 %}
10890
10891 ins_pipe(ialu_reg_reg_shift);
10892 %}
10893
10894 // This pattern is automatically generated from aarch64_ad.m4
10895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10896 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10897 iRegIorL2I src1, iRegIorL2I src2,
10898 immI src3) %{
10899 match(Set dst (XorI src1 (LShiftI src2 src3)));
10900
10901 ins_cost(1.9 * INSN_COST);
10902 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10903
10904 ins_encode %{
10905 __ eorw(as_Register($dst$$reg),
10906 as_Register($src1$$reg),
10907 as_Register($src2$$reg),
10908 Assembler::LSL,
10909 $src3$$constant & 0x1f);
10910 %}
10911
10912 ins_pipe(ialu_reg_reg_shift);
10913 %}
10914
10915 // This pattern is automatically generated from aarch64_ad.m4
10916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10917 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10918 iRegL src1, iRegL src2,
10919 immI src3) %{
10920 match(Set dst (XorL src1 (LShiftL src2 src3)));
10921
10922 ins_cost(1.9 * INSN_COST);
10923 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10924
10925 ins_encode %{
10926 __ eor(as_Register($dst$$reg),
10927 as_Register($src1$$reg),
10928 as_Register($src2$$reg),
10929 Assembler::LSL,
10930 $src3$$constant & 0x3f);
10931 %}
10932
10933 ins_pipe(ialu_reg_reg_shift);
10934 %}
10935
10936 // This pattern is automatically generated from aarch64_ad.m4
10937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10938 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10939 iRegIorL2I src1, iRegIorL2I src2,
10940 immI src3) %{
10941 match(Set dst (XorI src1 (RotateRight src2 src3)));
10942
10943 ins_cost(1.9 * INSN_COST);
10944 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10945
10946 ins_encode %{
10947 __ eorw(as_Register($dst$$reg),
10948 as_Register($src1$$reg),
10949 as_Register($src2$$reg),
10950 Assembler::ROR,
10951 $src3$$constant & 0x1f);
10952 %}
10953
10954 ins_pipe(ialu_reg_reg_shift);
10955 %}
10956
10957 // This pattern is automatically generated from aarch64_ad.m4
10958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10959 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10960 iRegL src1, iRegL src2,
10961 immI src3) %{
10962 match(Set dst (XorL src1 (RotateRight src2 src3)));
10963
10964 ins_cost(1.9 * INSN_COST);
10965 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10966
10967 ins_encode %{
10968 __ eor(as_Register($dst$$reg),
10969 as_Register($src1$$reg),
10970 as_Register($src2$$reg),
10971 Assembler::ROR,
10972 $src3$$constant & 0x3f);
10973 %}
10974
10975 ins_pipe(ialu_reg_reg_shift);
10976 %}
10977
10978 // This pattern is automatically generated from aarch64_ad.m4
10979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10980 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10981 iRegIorL2I src1, iRegIorL2I src2,
10982 immI src3) %{
10983 match(Set dst (OrI src1 (URShiftI src2 src3)));
10984
10985 ins_cost(1.9 * INSN_COST);
10986 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10987
10988 ins_encode %{
10989 __ orrw(as_Register($dst$$reg),
10990 as_Register($src1$$reg),
10991 as_Register($src2$$reg),
10992 Assembler::LSR,
10993 $src3$$constant & 0x1f);
10994 %}
10995
10996 ins_pipe(ialu_reg_reg_shift);
10997 %}
10998
10999 // This pattern is automatically generated from aarch64_ad.m4
11000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11001 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11002 iRegL src1, iRegL src2,
11003 immI src3) %{
11004 match(Set dst (OrL src1 (URShiftL src2 src3)));
11005
11006 ins_cost(1.9 * INSN_COST);
11007 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11008
11009 ins_encode %{
11010 __ orr(as_Register($dst$$reg),
11011 as_Register($src1$$reg),
11012 as_Register($src2$$reg),
11013 Assembler::LSR,
11014 $src3$$constant & 0x3f);
11015 %}
11016
11017 ins_pipe(ialu_reg_reg_shift);
11018 %}
11019
11020 // This pattern is automatically generated from aarch64_ad.m4
11021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11022 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11023 iRegIorL2I src1, iRegIorL2I src2,
11024 immI src3) %{
11025 match(Set dst (OrI src1 (RShiftI src2 src3)));
11026
11027 ins_cost(1.9 * INSN_COST);
11028 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11029
11030 ins_encode %{
11031 __ orrw(as_Register($dst$$reg),
11032 as_Register($src1$$reg),
11033 as_Register($src2$$reg),
11034 Assembler::ASR,
11035 $src3$$constant & 0x1f);
11036 %}
11037
11038 ins_pipe(ialu_reg_reg_shift);
11039 %}
11040
11041 // This pattern is automatically generated from aarch64_ad.m4
11042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11043 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11044 iRegL src1, iRegL src2,
11045 immI src3) %{
11046 match(Set dst (OrL src1 (RShiftL src2 src3)));
11047
11048 ins_cost(1.9 * INSN_COST);
11049 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11050
11051 ins_encode %{
11052 __ orr(as_Register($dst$$reg),
11053 as_Register($src1$$reg),
11054 as_Register($src2$$reg),
11055 Assembler::ASR,
11056 $src3$$constant & 0x3f);
11057 %}
11058
11059 ins_pipe(ialu_reg_reg_shift);
11060 %}
11061
11062 // This pattern is automatically generated from aarch64_ad.m4
11063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11064 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11065 iRegIorL2I src1, iRegIorL2I src2,
11066 immI src3) %{
11067 match(Set dst (OrI src1 (LShiftI src2 src3)));
11068
11069 ins_cost(1.9 * INSN_COST);
11070 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11071
11072 ins_encode %{
11073 __ orrw(as_Register($dst$$reg),
11074 as_Register($src1$$reg),
11075 as_Register($src2$$reg),
11076 Assembler::LSL,
11077 $src3$$constant & 0x1f);
11078 %}
11079
11080 ins_pipe(ialu_reg_reg_shift);
11081 %}
11082
11083 // This pattern is automatically generated from aarch64_ad.m4
11084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11085 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11086 iRegL src1, iRegL src2,
11087 immI src3) %{
11088 match(Set dst (OrL src1 (LShiftL src2 src3)));
11089
11090 ins_cost(1.9 * INSN_COST);
11091 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11092
11093 ins_encode %{
11094 __ orr(as_Register($dst$$reg),
11095 as_Register($src1$$reg),
11096 as_Register($src2$$reg),
11097 Assembler::LSL,
11098 $src3$$constant & 0x3f);
11099 %}
11100
11101 ins_pipe(ialu_reg_reg_shift);
11102 %}
11103
11104 // This pattern is automatically generated from aarch64_ad.m4
11105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11106 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11107 iRegIorL2I src1, iRegIorL2I src2,
11108 immI src3) %{
11109 match(Set dst (OrI src1 (RotateRight src2 src3)));
11110
11111 ins_cost(1.9 * INSN_COST);
11112 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11113
11114 ins_encode %{
11115 __ orrw(as_Register($dst$$reg),
11116 as_Register($src1$$reg),
11117 as_Register($src2$$reg),
11118 Assembler::ROR,
11119 $src3$$constant & 0x1f);
11120 %}
11121
11122 ins_pipe(ialu_reg_reg_shift);
11123 %}
11124
11125 // This pattern is automatically generated from aarch64_ad.m4
11126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11127 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11128 iRegL src1, iRegL src2,
11129 immI src3) %{
11130 match(Set dst (OrL src1 (RotateRight src2 src3)));
11131
11132 ins_cost(1.9 * INSN_COST);
11133 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11134
11135 ins_encode %{
11136 __ orr(as_Register($dst$$reg),
11137 as_Register($src1$$reg),
11138 as_Register($src2$$reg),
11139 Assembler::ROR,
11140 $src3$$constant & 0x3f);
11141 %}
11142
11143 ins_pipe(ialu_reg_reg_shift);
11144 %}
11145
11146 // This pattern is automatically generated from aarch64_ad.m4
11147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11148 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11149 iRegIorL2I src1, iRegIorL2I src2,
11150 immI src3) %{
11151 match(Set dst (AddI src1 (URShiftI src2 src3)));
11152
11153 ins_cost(1.9 * INSN_COST);
11154 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11155
11156 ins_encode %{
11157 __ addw(as_Register($dst$$reg),
11158 as_Register($src1$$reg),
11159 as_Register($src2$$reg),
11160 Assembler::LSR,
11161 $src3$$constant & 0x1f);
11162 %}
11163
11164 ins_pipe(ialu_reg_reg_shift);
11165 %}
11166
11167 // This pattern is automatically generated from aarch64_ad.m4
11168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11169 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11170 iRegL src1, iRegL src2,
11171 immI src3) %{
11172 match(Set dst (AddL src1 (URShiftL src2 src3)));
11173
11174 ins_cost(1.9 * INSN_COST);
11175 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11176
11177 ins_encode %{
11178 __ add(as_Register($dst$$reg),
11179 as_Register($src1$$reg),
11180 as_Register($src2$$reg),
11181 Assembler::LSR,
11182 $src3$$constant & 0x3f);
11183 %}
11184
11185 ins_pipe(ialu_reg_reg_shift);
11186 %}
11187
11188 // This pattern is automatically generated from aarch64_ad.m4
11189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11190 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11191 iRegIorL2I src1, iRegIorL2I src2,
11192 immI src3) %{
11193 match(Set dst (AddI src1 (RShiftI src2 src3)));
11194
11195 ins_cost(1.9 * INSN_COST);
11196 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11197
11198 ins_encode %{
11199 __ addw(as_Register($dst$$reg),
11200 as_Register($src1$$reg),
11201 as_Register($src2$$reg),
11202 Assembler::ASR,
11203 $src3$$constant & 0x1f);
11204 %}
11205
11206 ins_pipe(ialu_reg_reg_shift);
11207 %}
11208
11209 // This pattern is automatically generated from aarch64_ad.m4
11210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11211 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11212 iRegL src1, iRegL src2,
11213 immI src3) %{
11214 match(Set dst (AddL src1 (RShiftL src2 src3)));
11215
11216 ins_cost(1.9 * INSN_COST);
11217 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11218
11219 ins_encode %{
11220 __ add(as_Register($dst$$reg),
11221 as_Register($src1$$reg),
11222 as_Register($src2$$reg),
11223 Assembler::ASR,
11224 $src3$$constant & 0x3f);
11225 %}
11226
11227 ins_pipe(ialu_reg_reg_shift);
11228 %}
11229
11230 // This pattern is automatically generated from aarch64_ad.m4
11231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11232 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11233 iRegIorL2I src1, iRegIorL2I src2,
11234 immI src3) %{
11235 match(Set dst (AddI src1 (LShiftI src2 src3)));
11236
11237 ins_cost(1.9 * INSN_COST);
11238 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11239
11240 ins_encode %{
11241 __ addw(as_Register($dst$$reg),
11242 as_Register($src1$$reg),
11243 as_Register($src2$$reg),
11244 Assembler::LSL,
11245 $src3$$constant & 0x1f);
11246 %}
11247
11248 ins_pipe(ialu_reg_reg_shift);
11249 %}
11250
11251 // This pattern is automatically generated from aarch64_ad.m4
11252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11253 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11254 iRegL src1, iRegL src2,
11255 immI src3) %{
11256 match(Set dst (AddL src1 (LShiftL src2 src3)));
11257
11258 ins_cost(1.9 * INSN_COST);
11259 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11260
11261 ins_encode %{
11262 __ add(as_Register($dst$$reg),
11263 as_Register($src1$$reg),
11264 as_Register($src2$$reg),
11265 Assembler::LSL,
11266 $src3$$constant & 0x3f);
11267 %}
11268
11269 ins_pipe(ialu_reg_reg_shift);
11270 %}
11271
11272 // This pattern is automatically generated from aarch64_ad.m4
11273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11274 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11275 iRegIorL2I src1, iRegIorL2I src2,
11276 immI src3) %{
11277 match(Set dst (SubI src1 (URShiftI src2 src3)));
11278
11279 ins_cost(1.9 * INSN_COST);
11280 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11281
11282 ins_encode %{
11283 __ subw(as_Register($dst$$reg),
11284 as_Register($src1$$reg),
11285 as_Register($src2$$reg),
11286 Assembler::LSR,
11287 $src3$$constant & 0x1f);
11288 %}
11289
11290 ins_pipe(ialu_reg_reg_shift);
11291 %}
11292
11293 // This pattern is automatically generated from aarch64_ad.m4
11294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11295 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11296 iRegL src1, iRegL src2,
11297 immI src3) %{
11298 match(Set dst (SubL src1 (URShiftL src2 src3)));
11299
11300 ins_cost(1.9 * INSN_COST);
11301 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11302
11303 ins_encode %{
11304 __ sub(as_Register($dst$$reg),
11305 as_Register($src1$$reg),
11306 as_Register($src2$$reg),
11307 Assembler::LSR,
11308 $src3$$constant & 0x3f);
11309 %}
11310
11311 ins_pipe(ialu_reg_reg_shift);
11312 %}
11313
11314 // This pattern is automatically generated from aarch64_ad.m4
11315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11316 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11317 iRegIorL2I src1, iRegIorL2I src2,
11318 immI src3) %{
11319 match(Set dst (SubI src1 (RShiftI src2 src3)));
11320
11321 ins_cost(1.9 * INSN_COST);
11322 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11323
11324 ins_encode %{
11325 __ subw(as_Register($dst$$reg),
11326 as_Register($src1$$reg),
11327 as_Register($src2$$reg),
11328 Assembler::ASR,
11329 $src3$$constant & 0x1f);
11330 %}
11331
11332 ins_pipe(ialu_reg_reg_shift);
11333 %}
11334
11335 // This pattern is automatically generated from aarch64_ad.m4
11336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11337 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11338 iRegL src1, iRegL src2,
11339 immI src3) %{
11340 match(Set dst (SubL src1 (RShiftL src2 src3)));
11341
11342 ins_cost(1.9 * INSN_COST);
11343 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11344
11345 ins_encode %{
11346 __ sub(as_Register($dst$$reg),
11347 as_Register($src1$$reg),
11348 as_Register($src2$$reg),
11349 Assembler::ASR,
11350 $src3$$constant & 0x3f);
11351 %}
11352
11353 ins_pipe(ialu_reg_reg_shift);
11354 %}
11355
11356 // This pattern is automatically generated from aarch64_ad.m4
11357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11358 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11359 iRegIorL2I src1, iRegIorL2I src2,
11360 immI src3) %{
11361 match(Set dst (SubI src1 (LShiftI src2 src3)));
11362
11363 ins_cost(1.9 * INSN_COST);
11364 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11365
11366 ins_encode %{
11367 __ subw(as_Register($dst$$reg),
11368 as_Register($src1$$reg),
11369 as_Register($src2$$reg),
11370 Assembler::LSL,
11371 $src3$$constant & 0x1f);
11372 %}
11373
11374 ins_pipe(ialu_reg_reg_shift);
11375 %}
11376
11377 // This pattern is automatically generated from aarch64_ad.m4
11378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11379 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11380 iRegL src1, iRegL src2,
11381 immI src3) %{
11382 match(Set dst (SubL src1 (LShiftL src2 src3)));
11383
11384 ins_cost(1.9 * INSN_COST);
11385 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11386
11387 ins_encode %{
11388 __ sub(as_Register($dst$$reg),
11389 as_Register($src1$$reg),
11390 as_Register($src2$$reg),
11391 Assembler::LSL,
11392 $src3$$constant & 0x3f);
11393 %}
11394
11395 ins_pipe(ialu_reg_reg_shift);
11396 %}
11397
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400
11401 // Shift Left followed by Shift Right.
11402 // This idiom is used by the compiler for the i2b bytecode etc.
11403 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11404 %{
11405 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11406 ins_cost(INSN_COST * 2);
11407 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11408 ins_encode %{
11409 int lshift = $lshift_count$$constant & 63;
11410 int rshift = $rshift_count$$constant & 63;
11411 int s = 63 - lshift;
11412 int r = (rshift - lshift) & 63;
11413 __ sbfm(as_Register($dst$$reg),
11414 as_Register($src$$reg),
11415 r, s);
11416 %}
11417
11418 ins_pipe(ialu_reg_shift);
11419 %}
11420
11421 // This pattern is automatically generated from aarch64_ad.m4
11422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11423
11424 // Shift Left followed by Shift Right.
11425 // This idiom is used by the compiler for the i2b bytecode etc.
11426 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11427 %{
11428 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11429 ins_cost(INSN_COST * 2);
11430 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11431 ins_encode %{
11432 int lshift = $lshift_count$$constant & 31;
11433 int rshift = $rshift_count$$constant & 31;
11434 int s = 31 - lshift;
11435 int r = (rshift - lshift) & 31;
11436 __ sbfmw(as_Register($dst$$reg),
11437 as_Register($src$$reg),
11438 r, s);
11439 %}
11440
11441 ins_pipe(ialu_reg_shift);
11442 %}
11443
11444 // This pattern is automatically generated from aarch64_ad.m4
11445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11446
11447 // Shift Left followed by Shift Right.
11448 // This idiom is used by the compiler for the i2b bytecode etc.
11449 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11450 %{
11451 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11452 ins_cost(INSN_COST * 2);
11453 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11454 ins_encode %{
11455 int lshift = $lshift_count$$constant & 63;
11456 int rshift = $rshift_count$$constant & 63;
11457 int s = 63 - lshift;
11458 int r = (rshift - lshift) & 63;
11459 __ ubfm(as_Register($dst$$reg),
11460 as_Register($src$$reg),
11461 r, s);
11462 %}
11463
11464 ins_pipe(ialu_reg_shift);
11465 %}
11466
11467 // This pattern is automatically generated from aarch64_ad.m4
11468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11469
11470 // Shift Left followed by Shift Right.
11471 // This idiom is used by the compiler for the i2b bytecode etc.
11472 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11473 %{
11474 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11475 ins_cost(INSN_COST * 2);
11476 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11477 ins_encode %{
11478 int lshift = $lshift_count$$constant & 31;
11479 int rshift = $rshift_count$$constant & 31;
11480 int s = 31 - lshift;
11481 int r = (rshift - lshift) & 31;
11482 __ ubfmw(as_Register($dst$$reg),
11483 as_Register($src$$reg),
11484 r, s);
11485 %}
11486
11487 ins_pipe(ialu_reg_shift);
11488 %}
11489
11490 // Bitfield extract with shift & mask
11491
11492 // This pattern is automatically generated from aarch64_ad.m4
11493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11494 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11495 %{
11496 match(Set dst (AndI (URShiftI src rshift) mask));
11497 // Make sure we are not going to exceed what ubfxw can do.
11498 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11499
11500 ins_cost(INSN_COST);
11501 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11502 ins_encode %{
11503 int rshift = $rshift$$constant & 31;
11504 intptr_t mask = $mask$$constant;
11505 int width = exact_log2(mask+1);
11506 __ ubfxw(as_Register($dst$$reg),
11507 as_Register($src$$reg), rshift, width);
11508 %}
11509 ins_pipe(ialu_reg_shift);
11510 %}
11511
11512 // This pattern is automatically generated from aarch64_ad.m4
11513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11514 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11515 %{
11516 match(Set dst (AndL (URShiftL src rshift) mask));
11517 // Make sure we are not going to exceed what ubfx can do.
11518 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11519
11520 ins_cost(INSN_COST);
11521 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11522 ins_encode %{
11523 int rshift = $rshift$$constant & 63;
11524 intptr_t mask = $mask$$constant;
11525 int width = exact_log2_long(mask+1);
11526 __ ubfx(as_Register($dst$$reg),
11527 as_Register($src$$reg), rshift, width);
11528 %}
11529 ins_pipe(ialu_reg_shift);
11530 %}
11531
11532
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535
11536 // We can use ubfx when extending an And with a mask when we know mask
11537 // is positive. We know that because immI_bitmask guarantees it.
11538 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11539 %{
11540 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11541 // Make sure we are not going to exceed what ubfxw can do.
11542 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11543
11544 ins_cost(INSN_COST * 2);
11545 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11546 ins_encode %{
11547 int rshift = $rshift$$constant & 31;
11548 intptr_t mask = $mask$$constant;
11549 int width = exact_log2(mask+1);
11550 __ ubfx(as_Register($dst$$reg),
11551 as_Register($src$$reg), rshift, width);
11552 %}
11553 ins_pipe(ialu_reg_shift);
11554 %}
11555
11556
11557 // This pattern is automatically generated from aarch64_ad.m4
11558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11559
11560 // We can use ubfiz when masking by a positive number and then left shifting the result.
11561 // We know that the mask is positive because immI_bitmask guarantees it.
11562 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11563 %{
11564 match(Set dst (LShiftI (AndI src mask) lshift));
11565 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11566
11567 ins_cost(INSN_COST);
11568 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11569 ins_encode %{
11570 int lshift = $lshift$$constant & 31;
11571 intptr_t mask = $mask$$constant;
11572 int width = exact_log2(mask+1);
11573 __ ubfizw(as_Register($dst$$reg),
11574 as_Register($src$$reg), lshift, width);
11575 %}
11576 ins_pipe(ialu_reg_shift);
11577 %}
11578
11579 // This pattern is automatically generated from aarch64_ad.m4
11580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11581
11582 // We can use ubfiz when masking by a positive number and then left shifting the result.
11583 // We know that the mask is positive because immL_bitmask guarantees it.
11584 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11585 %{
11586 match(Set dst (LShiftL (AndL src mask) lshift));
11587 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11588
11589 ins_cost(INSN_COST);
11590 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11591 ins_encode %{
11592 int lshift = $lshift$$constant & 63;
11593 intptr_t mask = $mask$$constant;
11594 int width = exact_log2_long(mask+1);
11595 __ ubfiz(as_Register($dst$$reg),
11596 as_Register($src$$reg), lshift, width);
11597 %}
11598 ins_pipe(ialu_reg_shift);
11599 %}
11600
11601 // This pattern is automatically generated from aarch64_ad.m4
11602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11603
11604 // We can use ubfiz when masking by a positive number and then left shifting the result.
11605 // We know that the mask is positive because immI_bitmask guarantees it.
11606 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11607 %{
11608 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11609 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11610
11611 ins_cost(INSN_COST);
11612 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11613 ins_encode %{
11614 int lshift = $lshift$$constant & 31;
11615 intptr_t mask = $mask$$constant;
11616 int width = exact_log2(mask+1);
11617 __ ubfizw(as_Register($dst$$reg),
11618 as_Register($src$$reg), lshift, width);
11619 %}
11620 ins_pipe(ialu_reg_shift);
11621 %}
11622
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625
11626 // We can use ubfiz when masking by a positive number and then left shifting the result.
11627 // We know that the mask is positive because immL_bitmask guarantees it.
11628 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11629 %{
11630 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11631 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11632
11633 ins_cost(INSN_COST);
11634 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11635 ins_encode %{
11636 int lshift = $lshift$$constant & 63;
11637 intptr_t mask = $mask$$constant;
11638 int width = exact_log2_long(mask+1);
11639 __ ubfiz(as_Register($dst$$reg),
11640 as_Register($src$$reg), lshift, width);
11641 %}
11642 ins_pipe(ialu_reg_shift);
11643 %}
11644
11645
11646 // This pattern is automatically generated from aarch64_ad.m4
11647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11648
11649 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11650 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11651 %{
11652 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11653 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11654
11655 ins_cost(INSN_COST);
11656 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11657 ins_encode %{
11658 int lshift = $lshift$$constant & 63;
11659 intptr_t mask = $mask$$constant;
11660 int width = exact_log2(mask+1);
11661 __ ubfiz(as_Register($dst$$reg),
11662 as_Register($src$$reg), lshift, width);
11663 %}
11664 ins_pipe(ialu_reg_shift);
11665 %}
11666
11667 // This pattern is automatically generated from aarch64_ad.m4
11668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11669
11670 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11671 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11672 %{
11673 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11674 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11675
11676 ins_cost(INSN_COST);
11677 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11678 ins_encode %{
11679 int lshift = $lshift$$constant & 31;
11680 intptr_t mask = $mask$$constant;
11681 int width = exact_log2(mask+1);
11682 __ ubfiz(as_Register($dst$$reg),
11683 as_Register($src$$reg), lshift, width);
11684 %}
11685 ins_pipe(ialu_reg_shift);
11686 %}
11687
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690
11691 // Can skip int2long conversions after AND with small bitmask
11692 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11693 %{
11694 match(Set dst (ConvI2L (AndI src msk)));
11695 ins_cost(INSN_COST);
11696 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11697 ins_encode %{
11698 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11699 %}
11700 ins_pipe(ialu_reg_shift);
11701 %}
11702
11703
11704 // Rotations
11705
11706 // This pattern is automatically generated from aarch64_ad.m4
11707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11708 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11709 %{
11710 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11711 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11712
11713 ins_cost(INSN_COST);
11714 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11715
11716 ins_encode %{
11717 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11718 $rshift$$constant & 63);
11719 %}
11720 ins_pipe(ialu_reg_reg_extr);
11721 %}
11722
11723
11724 // This pattern is automatically generated from aarch64_ad.m4
11725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11726 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11727 %{
11728 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11729 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11730
11731 ins_cost(INSN_COST);
11732 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11733
11734 ins_encode %{
11735 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11736 $rshift$$constant & 31);
11737 %}
11738 ins_pipe(ialu_reg_reg_extr);
11739 %}
11740
11741
11742 // This pattern is automatically generated from aarch64_ad.m4
11743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11744 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11745 %{
11746 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11747 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11748
11749 ins_cost(INSN_COST);
11750 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11751
11752 ins_encode %{
11753 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11754 $rshift$$constant & 63);
11755 %}
11756 ins_pipe(ialu_reg_reg_extr);
11757 %}
11758
11759
11760 // This pattern is automatically generated from aarch64_ad.m4
11761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11762 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11763 %{
11764 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11765 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11766
11767 ins_cost(INSN_COST);
11768 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11769
11770 ins_encode %{
11771 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11772 $rshift$$constant & 31);
11773 %}
11774 ins_pipe(ialu_reg_reg_extr);
11775 %}
11776
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11780 %{
11781 match(Set dst (RotateRight src shift));
11782
11783 ins_cost(INSN_COST);
11784 format %{ "ror $dst, $src, $shift" %}
11785
11786 ins_encode %{
11787 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11788 $shift$$constant & 0x1f);
11789 %}
11790 ins_pipe(ialu_reg_reg_vshift);
11791 %}
11792
11793 // This pattern is automatically generated from aarch64_ad.m4
11794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11795 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11796 %{
11797 match(Set dst (RotateRight src shift));
11798
11799 ins_cost(INSN_COST);
11800 format %{ "ror $dst, $src, $shift" %}
11801
11802 ins_encode %{
11803 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11804 $shift$$constant & 0x3f);
11805 %}
11806 ins_pipe(ialu_reg_reg_vshift);
11807 %}
11808
11809 // This pattern is automatically generated from aarch64_ad.m4
11810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11811 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11812 %{
11813 match(Set dst (RotateRight src shift));
11814
11815 ins_cost(INSN_COST);
11816 format %{ "ror $dst, $src, $shift" %}
11817
11818 ins_encode %{
11819 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11820 %}
11821 ins_pipe(ialu_reg_reg_vshift);
11822 %}
11823
11824 // This pattern is automatically generated from aarch64_ad.m4
11825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11826 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11827 %{
11828 match(Set dst (RotateRight src shift));
11829
11830 ins_cost(INSN_COST);
11831 format %{ "ror $dst, $src, $shift" %}
11832
11833 ins_encode %{
11834 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11835 %}
11836 ins_pipe(ialu_reg_reg_vshift);
11837 %}
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11842 %{
11843 match(Set dst (RotateLeft src shift));
11844
11845 ins_cost(INSN_COST);
11846 format %{ "rol $dst, $src, $shift" %}
11847
11848 ins_encode %{
11849 __ subw(rscratch1, zr, as_Register($shift$$reg));
11850 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11851 %}
11852 ins_pipe(ialu_reg_reg_vshift);
11853 %}
11854
11855 // This pattern is automatically generated from aarch64_ad.m4
11856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11857 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11858 %{
11859 match(Set dst (RotateLeft src shift));
11860
11861 ins_cost(INSN_COST);
11862 format %{ "rol $dst, $src, $shift" %}
11863
11864 ins_encode %{
11865 __ subw(rscratch1, zr, as_Register($shift$$reg));
11866 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11867 %}
11868 ins_pipe(ialu_reg_reg_vshift);
11869 %}
11870
11871
11872 // Add/subtract (extended)
11873
11874 // This pattern is automatically generated from aarch64_ad.m4
11875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11876 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11877 %{
11878 match(Set dst (AddL src1 (ConvI2L src2)));
11879 ins_cost(INSN_COST);
11880 format %{ "add $dst, $src1, $src2, sxtw" %}
11881
11882 ins_encode %{
11883 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11884 as_Register($src2$$reg), ext::sxtw);
11885 %}
11886 ins_pipe(ialu_reg_reg);
11887 %}
11888
11889 // This pattern is automatically generated from aarch64_ad.m4
11890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11891 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11892 %{
11893 match(Set dst (SubL src1 (ConvI2L src2)));
11894 ins_cost(INSN_COST);
11895 format %{ "sub $dst, $src1, $src2, sxtw" %}
11896
11897 ins_encode %{
11898 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11899 as_Register($src2$$reg), ext::sxtw);
11900 %}
11901 ins_pipe(ialu_reg_reg);
11902 %}
11903
11904 // This pattern is automatically generated from aarch64_ad.m4
11905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11906 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11907 %{
11908 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11909 ins_cost(INSN_COST);
11910 format %{ "add $dst, $src1, $src2, sxth" %}
11911
11912 ins_encode %{
11913 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11914 as_Register($src2$$reg), ext::sxth);
11915 %}
11916 ins_pipe(ialu_reg_reg);
11917 %}
11918
11919 // This pattern is automatically generated from aarch64_ad.m4
11920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11921 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11922 %{
11923 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11924 ins_cost(INSN_COST);
11925 format %{ "add $dst, $src1, $src2, sxtb" %}
11926
11927 ins_encode %{
11928 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11929 as_Register($src2$$reg), ext::sxtb);
11930 %}
11931 ins_pipe(ialu_reg_reg);
11932 %}
11933
11934 // This pattern is automatically generated from aarch64_ad.m4
11935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11936 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11937 %{
11938 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11939 ins_cost(INSN_COST);
11940 format %{ "add $dst, $src1, $src2, uxtb" %}
11941
11942 ins_encode %{
11943 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11944 as_Register($src2$$reg), ext::uxtb);
11945 %}
11946 ins_pipe(ialu_reg_reg);
11947 %}
11948
11949 // This pattern is automatically generated from aarch64_ad.m4
11950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11951 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11952 %{
11953 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11954 ins_cost(INSN_COST);
11955 format %{ "add $dst, $src1, $src2, sxth" %}
11956
11957 ins_encode %{
11958 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11959 as_Register($src2$$reg), ext::sxth);
11960 %}
11961 ins_pipe(ialu_reg_reg);
11962 %}
11963
11964 // This pattern is automatically generated from aarch64_ad.m4
11965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11966 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11967 %{
11968 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11969 ins_cost(INSN_COST);
11970 format %{ "add $dst, $src1, $src2, sxtw" %}
11971
11972 ins_encode %{
11973 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11974 as_Register($src2$$reg), ext::sxtw);
11975 %}
11976 ins_pipe(ialu_reg_reg);
11977 %}
11978
11979 // This pattern is automatically generated from aarch64_ad.m4
11980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11981 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11982 %{
11983 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11984 ins_cost(INSN_COST);
11985 format %{ "add $dst, $src1, $src2, sxtb" %}
11986
11987 ins_encode %{
11988 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11989 as_Register($src2$$reg), ext::sxtb);
11990 %}
11991 ins_pipe(ialu_reg_reg);
11992 %}
11993
11994 // This pattern is automatically generated from aarch64_ad.m4
11995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11996 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11997 %{
11998 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11999 ins_cost(INSN_COST);
12000 format %{ "add $dst, $src1, $src2, uxtb" %}
12001
12002 ins_encode %{
12003 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12004 as_Register($src2$$reg), ext::uxtb);
12005 %}
12006 ins_pipe(ialu_reg_reg);
12007 %}
12008
12009 // This pattern is automatically generated from aarch64_ad.m4
12010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12011 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12012 %{
12013 match(Set dst (AddI src1 (AndI src2 mask)));
12014 ins_cost(INSN_COST);
12015 format %{ "addw $dst, $src1, $src2, uxtb" %}
12016
12017 ins_encode %{
12018 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12019 as_Register($src2$$reg), ext::uxtb);
12020 %}
12021 ins_pipe(ialu_reg_reg);
12022 %}
12023
12024 // This pattern is automatically generated from aarch64_ad.m4
12025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12026 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12027 %{
12028 match(Set dst (AddI src1 (AndI src2 mask)));
12029 ins_cost(INSN_COST);
12030 format %{ "addw $dst, $src1, $src2, uxth" %}
12031
12032 ins_encode %{
12033 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12034 as_Register($src2$$reg), ext::uxth);
12035 %}
12036 ins_pipe(ialu_reg_reg);
12037 %}
12038
12039 // This pattern is automatically generated from aarch64_ad.m4
12040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12041 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12042 %{
12043 match(Set dst (AddL src1 (AndL src2 mask)));
12044 ins_cost(INSN_COST);
12045 format %{ "add $dst, $src1, $src2, uxtb" %}
12046
12047 ins_encode %{
12048 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12049 as_Register($src2$$reg), ext::uxtb);
12050 %}
12051 ins_pipe(ialu_reg_reg);
12052 %}
12053
12054 // This pattern is automatically generated from aarch64_ad.m4
12055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12056 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12057 %{
12058 match(Set dst (AddL src1 (AndL src2 mask)));
12059 ins_cost(INSN_COST);
12060 format %{ "add $dst, $src1, $src2, uxth" %}
12061
12062 ins_encode %{
12063 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12064 as_Register($src2$$reg), ext::uxth);
12065 %}
12066 ins_pipe(ialu_reg_reg);
12067 %}
12068
12069 // This pattern is automatically generated from aarch64_ad.m4
12070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12071 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12072 %{
12073 match(Set dst (AddL src1 (AndL src2 mask)));
12074 ins_cost(INSN_COST);
12075 format %{ "add $dst, $src1, $src2, uxtw" %}
12076
12077 ins_encode %{
12078 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12079 as_Register($src2$$reg), ext::uxtw);
12080 %}
12081 ins_pipe(ialu_reg_reg);
12082 %}
12083
12084 // This pattern is automatically generated from aarch64_ad.m4
12085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12086 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12087 %{
12088 match(Set dst (SubI src1 (AndI src2 mask)));
12089 ins_cost(INSN_COST);
12090 format %{ "subw $dst, $src1, $src2, uxtb" %}
12091
12092 ins_encode %{
12093 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12094 as_Register($src2$$reg), ext::uxtb);
12095 %}
12096 ins_pipe(ialu_reg_reg);
12097 %}
12098
12099 // This pattern is automatically generated from aarch64_ad.m4
12100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12101 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12102 %{
12103 match(Set dst (SubI src1 (AndI src2 mask)));
12104 ins_cost(INSN_COST);
12105 format %{ "subw $dst, $src1, $src2, uxth" %}
12106
12107 ins_encode %{
12108 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12109 as_Register($src2$$reg), ext::uxth);
12110 %}
12111 ins_pipe(ialu_reg_reg);
12112 %}
12113
12114 // This pattern is automatically generated from aarch64_ad.m4
12115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12116 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12117 %{
12118 match(Set dst (SubL src1 (AndL src2 mask)));
12119 ins_cost(INSN_COST);
12120 format %{ "sub $dst, $src1, $src2, uxtb" %}
12121
12122 ins_encode %{
12123 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12124 as_Register($src2$$reg), ext::uxtb);
12125 %}
12126 ins_pipe(ialu_reg_reg);
12127 %}
12128
12129 // This pattern is automatically generated from aarch64_ad.m4
12130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12131 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12132 %{
12133 match(Set dst (SubL src1 (AndL src2 mask)));
12134 ins_cost(INSN_COST);
12135 format %{ "sub $dst, $src1, $src2, uxth" %}
12136
12137 ins_encode %{
12138 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12139 as_Register($src2$$reg), ext::uxth);
12140 %}
12141 ins_pipe(ialu_reg_reg);
12142 %}
12143
12144 // This pattern is automatically generated from aarch64_ad.m4
12145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12146 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12147 %{
12148 match(Set dst (SubL src1 (AndL src2 mask)));
12149 ins_cost(INSN_COST);
12150 format %{ "sub $dst, $src1, $src2, uxtw" %}
12151
12152 ins_encode %{
12153 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12154 as_Register($src2$$reg), ext::uxtw);
12155 %}
12156 ins_pipe(ialu_reg_reg);
12157 %}
12158
12159
12160 // This pattern is automatically generated from aarch64_ad.m4
12161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12162 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12163 %{
12164 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12165 ins_cost(1.9 * INSN_COST);
12166 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12167
12168 ins_encode %{
12169 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12170 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12171 %}
12172 ins_pipe(ialu_reg_reg_shift);
12173 %}
12174
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12178 %{
12179 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12180 ins_cost(1.9 * INSN_COST);
12181 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12182
12183 ins_encode %{
12184 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12185 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12186 %}
12187 ins_pipe(ialu_reg_reg_shift);
12188 %}
12189
12190 // This pattern is automatically generated from aarch64_ad.m4
12191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12192 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12193 %{
12194 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12195 ins_cost(1.9 * INSN_COST);
12196 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12197
12198 ins_encode %{
12199 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12200 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12201 %}
12202 ins_pipe(ialu_reg_reg_shift);
12203 %}
12204
12205 // This pattern is automatically generated from aarch64_ad.m4
12206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12207 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12208 %{
12209 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12210 ins_cost(1.9 * INSN_COST);
12211 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12212
12213 ins_encode %{
12214 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12215 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12216 %}
12217 ins_pipe(ialu_reg_reg_shift);
12218 %}
12219
12220 // This pattern is automatically generated from aarch64_ad.m4
12221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12222 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12223 %{
12224 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12225 ins_cost(1.9 * INSN_COST);
12226 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12227
12228 ins_encode %{
12229 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12230 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12231 %}
12232 ins_pipe(ialu_reg_reg_shift);
12233 %}
12234
12235 // This pattern is automatically generated from aarch64_ad.m4
12236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12237 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12238 %{
12239 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12240 ins_cost(1.9 * INSN_COST);
12241 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12242
12243 ins_encode %{
12244 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12245 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12246 %}
12247 ins_pipe(ialu_reg_reg_shift);
12248 %}
12249
12250 // This pattern is automatically generated from aarch64_ad.m4
12251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12252 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12253 %{
12254 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12255 ins_cost(1.9 * INSN_COST);
12256 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12257
12258 ins_encode %{
12259 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12260 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12261 %}
12262 ins_pipe(ialu_reg_reg_shift);
12263 %}
12264
12265 // This pattern is automatically generated from aarch64_ad.m4
12266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12267 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12268 %{
12269 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12270 ins_cost(1.9 * INSN_COST);
12271 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12272
12273 ins_encode %{
12274 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12275 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12276 %}
12277 ins_pipe(ialu_reg_reg_shift);
12278 %}
12279
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12283 %{
12284 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12285 ins_cost(1.9 * INSN_COST);
12286 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12287
12288 ins_encode %{
12289 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12290 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12291 %}
12292 ins_pipe(ialu_reg_reg_shift);
12293 %}
12294
12295 // This pattern is automatically generated from aarch64_ad.m4
12296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12297 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12298 %{
12299 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12300 ins_cost(1.9 * INSN_COST);
12301 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12302
12303 ins_encode %{
12304 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12305 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12306 %}
12307 ins_pipe(ialu_reg_reg_shift);
12308 %}
12309
12310 // This pattern is automatically generated from aarch64_ad.m4
12311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12312 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12313 %{
12314 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12315 ins_cost(1.9 * INSN_COST);
12316 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12317
12318 ins_encode %{
12319 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12320 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12321 %}
12322 ins_pipe(ialu_reg_reg_shift);
12323 %}
12324
12325 // This pattern is automatically generated from aarch64_ad.m4
12326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12327 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12328 %{
12329 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12330 ins_cost(1.9 * INSN_COST);
12331 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12332
12333 ins_encode %{
12334 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12335 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12336 %}
12337 ins_pipe(ialu_reg_reg_shift);
12338 %}
12339
12340 // This pattern is automatically generated from aarch64_ad.m4
12341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12342 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12343 %{
12344 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12345 ins_cost(1.9 * INSN_COST);
12346 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12347
12348 ins_encode %{
12349 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12350 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12351 %}
12352 ins_pipe(ialu_reg_reg_shift);
12353 %}
12354
12355 // This pattern is automatically generated from aarch64_ad.m4
12356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12357 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12358 %{
12359 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12360 ins_cost(1.9 * INSN_COST);
12361 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12362
12363 ins_encode %{
12364 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12365 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12366 %}
12367 ins_pipe(ialu_reg_reg_shift);
12368 %}
12369
12370 // This pattern is automatically generated from aarch64_ad.m4
12371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12372 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12373 %{
12374 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12375 ins_cost(1.9 * INSN_COST);
12376 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12377
12378 ins_encode %{
12379 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12380 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12381 %}
12382 ins_pipe(ialu_reg_reg_shift);
12383 %}
12384
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12388 %{
12389 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12390 ins_cost(1.9 * INSN_COST);
12391 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12392
12393 ins_encode %{
12394 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12395 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12396 %}
12397 ins_pipe(ialu_reg_reg_shift);
12398 %}
12399
12400 // This pattern is automatically generated from aarch64_ad.m4
12401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12402 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12403 %{
12404 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12405 ins_cost(1.9 * INSN_COST);
12406 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12407
12408 ins_encode %{
12409 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12410 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12411 %}
12412 ins_pipe(ialu_reg_reg_shift);
12413 %}
12414
12415 // This pattern is automatically generated from aarch64_ad.m4
12416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12417 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12418 %{
12419 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12420 ins_cost(1.9 * INSN_COST);
12421 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12422
12423 ins_encode %{
12424 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12425 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12426 %}
12427 ins_pipe(ialu_reg_reg_shift);
12428 %}
12429
12430 // This pattern is automatically generated from aarch64_ad.m4
12431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12432 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12433 %{
12434 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12435 ins_cost(1.9 * INSN_COST);
12436 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12437
12438 ins_encode %{
12439 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12440 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12441 %}
12442 ins_pipe(ialu_reg_reg_shift);
12443 %}
12444
12445 // This pattern is automatically generated from aarch64_ad.m4
12446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12447 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12448 %{
12449 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12450 ins_cost(1.9 * INSN_COST);
12451 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12452
12453 ins_encode %{
12454 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12455 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12456 %}
12457 ins_pipe(ialu_reg_reg_shift);
12458 %}
12459
12460 // This pattern is automatically generated from aarch64_ad.m4
12461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12462 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12463 %{
12464 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12465 ins_cost(1.9 * INSN_COST);
12466 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12467
12468 ins_encode %{
12469 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12470 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12471 %}
12472 ins_pipe(ialu_reg_reg_shift);
12473 %}
12474
12475 // This pattern is automatically generated from aarch64_ad.m4
12476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12477 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12478 %{
12479 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12480 ins_cost(1.9 * INSN_COST);
12481 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12482
12483 ins_encode %{
12484 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12485 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12486 %}
12487 ins_pipe(ialu_reg_reg_shift);
12488 %}
12489
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12493 %{
12494 effect(DEF dst, USE src1, USE src2, USE cr);
12495 ins_cost(INSN_COST * 2);
12496 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12497
12498 ins_encode %{
12499 __ cselw($dst$$Register,
12500 $src1$$Register,
12501 $src2$$Register,
12502 Assembler::LT);
12503 %}
12504 ins_pipe(icond_reg_reg);
12505 %}
12506
12507 // This pattern is automatically generated from aarch64_ad.m4
12508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12509 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12510 %{
12511 effect(DEF dst, USE src1, USE src2, USE cr);
12512 ins_cost(INSN_COST * 2);
12513 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12514
12515 ins_encode %{
12516 __ cselw($dst$$Register,
12517 $src1$$Register,
12518 $src2$$Register,
12519 Assembler::GT);
12520 %}
12521 ins_pipe(icond_reg_reg);
12522 %}
12523
12524 // This pattern is automatically generated from aarch64_ad.m4
12525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12526 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12527 %{
12528 effect(DEF dst, USE src1, USE cr);
12529 ins_cost(INSN_COST * 2);
12530 format %{ "cselw $dst, $src1, zr lt\t" %}
12531
12532 ins_encode %{
12533 __ cselw($dst$$Register,
12534 $src1$$Register,
12535 zr,
12536 Assembler::LT);
12537 %}
12538 ins_pipe(icond_reg);
12539 %}
12540
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12544 %{
12545 effect(DEF dst, USE src1, USE cr);
12546 ins_cost(INSN_COST * 2);
12547 format %{ "cselw $dst, $src1, zr gt\t" %}
12548
12549 ins_encode %{
12550 __ cselw($dst$$Register,
12551 $src1$$Register,
12552 zr,
12553 Assembler::GT);
12554 %}
12555 ins_pipe(icond_reg);
12556 %}
12557
12558 // This pattern is automatically generated from aarch64_ad.m4
12559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12560 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12561 %{
12562 effect(DEF dst, USE src1, USE cr);
12563 ins_cost(INSN_COST * 2);
12564 format %{ "csincw $dst, $src1, zr le\t" %}
12565
12566 ins_encode %{
12567 __ csincw($dst$$Register,
12568 $src1$$Register,
12569 zr,
12570 Assembler::LE);
12571 %}
12572 ins_pipe(icond_reg);
12573 %}
12574
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12578 %{
12579 effect(DEF dst, USE src1, USE cr);
12580 ins_cost(INSN_COST * 2);
12581 format %{ "csincw $dst, $src1, zr gt\t" %}
12582
12583 ins_encode %{
12584 __ csincw($dst$$Register,
12585 $src1$$Register,
12586 zr,
12587 Assembler::GT);
12588 %}
12589 ins_pipe(icond_reg);
12590 %}
12591
12592 // This pattern is automatically generated from aarch64_ad.m4
12593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12594 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12595 %{
12596 effect(DEF dst, USE src1, USE cr);
12597 ins_cost(INSN_COST * 2);
12598 format %{ "csinvw $dst, $src1, zr lt\t" %}
12599
12600 ins_encode %{
12601 __ csinvw($dst$$Register,
12602 $src1$$Register,
12603 zr,
12604 Assembler::LT);
12605 %}
12606 ins_pipe(icond_reg);
12607 %}
12608
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12612 %{
12613 effect(DEF dst, USE src1, USE cr);
12614 ins_cost(INSN_COST * 2);
12615 format %{ "csinvw $dst, $src1, zr ge\t" %}
12616
12617 ins_encode %{
12618 __ csinvw($dst$$Register,
12619 $src1$$Register,
12620 zr,
12621 Assembler::GE);
12622 %}
12623 ins_pipe(icond_reg);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12629 %{
12630 match(Set dst (MinI src imm));
12631 ins_cost(INSN_COST * 3);
12632 expand %{
12633 rFlagsReg cr;
12634 compI_reg_imm0(cr, src);
12635 cmovI_reg_imm0_lt(dst, src, cr);
12636 %}
12637 %}
12638
12639 // This pattern is automatically generated from aarch64_ad.m4
12640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12641 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12642 %{
12643 match(Set dst (MinI imm src));
12644 ins_cost(INSN_COST * 3);
12645 expand %{
12646 rFlagsReg cr;
12647 compI_reg_imm0(cr, src);
12648 cmovI_reg_imm0_lt(dst, src, cr);
12649 %}
12650 %}
12651
12652 // This pattern is automatically generated from aarch64_ad.m4
12653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12654 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12655 %{
12656 match(Set dst (MinI src imm));
12657 ins_cost(INSN_COST * 3);
12658 expand %{
12659 rFlagsReg cr;
12660 compI_reg_imm0(cr, src);
12661 cmovI_reg_imm1_le(dst, src, cr);
12662 %}
12663 %}
12664
12665 // This pattern is automatically generated from aarch64_ad.m4
12666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12667 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12668 %{
12669 match(Set dst (MinI imm src));
12670 ins_cost(INSN_COST * 3);
12671 expand %{
12672 rFlagsReg cr;
12673 compI_reg_imm0(cr, src);
12674 cmovI_reg_imm1_le(dst, src, cr);
12675 %}
12676 %}
12677
12678 // This pattern is automatically generated from aarch64_ad.m4
12679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12680 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12681 %{
12682 match(Set dst (MinI src imm));
12683 ins_cost(INSN_COST * 3);
12684 expand %{
12685 rFlagsReg cr;
12686 compI_reg_imm0(cr, src);
12687 cmovI_reg_immM1_lt(dst, src, cr);
12688 %}
12689 %}
12690
12691 // This pattern is automatically generated from aarch64_ad.m4
12692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12693 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12694 %{
12695 match(Set dst (MinI imm src));
12696 ins_cost(INSN_COST * 3);
12697 expand %{
12698 rFlagsReg cr;
12699 compI_reg_imm0(cr, src);
12700 cmovI_reg_immM1_lt(dst, src, cr);
12701 %}
12702 %}
12703
12704 // This pattern is automatically generated from aarch64_ad.m4
12705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12706 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12707 %{
12708 match(Set dst (MaxI src imm));
12709 ins_cost(INSN_COST * 3);
12710 expand %{
12711 rFlagsReg cr;
12712 compI_reg_imm0(cr, src);
12713 cmovI_reg_imm0_gt(dst, src, cr);
12714 %}
12715 %}
12716
12717 // This pattern is automatically generated from aarch64_ad.m4
12718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12719 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12720 %{
12721 match(Set dst (MaxI imm src));
12722 ins_cost(INSN_COST * 3);
12723 expand %{
12724 rFlagsReg cr;
12725 compI_reg_imm0(cr, src);
12726 cmovI_reg_imm0_gt(dst, src, cr);
12727 %}
12728 %}
12729
12730 // This pattern is automatically generated from aarch64_ad.m4
12731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12732 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12733 %{
12734 match(Set dst (MaxI src imm));
12735 ins_cost(INSN_COST * 3);
12736 expand %{
12737 rFlagsReg cr;
12738 compI_reg_imm0(cr, src);
12739 cmovI_reg_imm1_gt(dst, src, cr);
12740 %}
12741 %}
12742
12743 // This pattern is automatically generated from aarch64_ad.m4
12744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12745 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12746 %{
12747 match(Set dst (MaxI imm src));
12748 ins_cost(INSN_COST * 3);
12749 expand %{
12750 rFlagsReg cr;
12751 compI_reg_imm0(cr, src);
12752 cmovI_reg_imm1_gt(dst, src, cr);
12753 %}
12754 %}
12755
12756 // This pattern is automatically generated from aarch64_ad.m4
12757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12758 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12759 %{
12760 match(Set dst (MaxI src imm));
12761 ins_cost(INSN_COST * 3);
12762 expand %{
12763 rFlagsReg cr;
12764 compI_reg_imm0(cr, src);
12765 cmovI_reg_immM1_ge(dst, src, cr);
12766 %}
12767 %}
12768
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12772 %{
12773 match(Set dst (MaxI imm src));
12774 ins_cost(INSN_COST * 3);
12775 expand %{
12776 rFlagsReg cr;
12777 compI_reg_imm0(cr, src);
12778 cmovI_reg_immM1_ge(dst, src, cr);
12779 %}
12780 %}
12781
12782 // This pattern is automatically generated from aarch64_ad.m4
12783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12784 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12785 %{
12786 match(Set dst (ReverseI src));
12787 ins_cost(INSN_COST);
12788 format %{ "rbitw $dst, $src" %}
12789 ins_encode %{
12790 __ rbitw($dst$$Register, $src$$Register);
12791 %}
12792 ins_pipe(ialu_reg);
12793 %}
12794
12795 // This pattern is automatically generated from aarch64_ad.m4
12796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12797 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12798 %{
12799 match(Set dst (ReverseL src));
12800 ins_cost(INSN_COST);
12801 format %{ "rbit $dst, $src" %}
12802 ins_encode %{
12803 __ rbit($dst$$Register, $src$$Register);
12804 %}
12805 ins_pipe(ialu_reg);
12806 %}
12807
12808
12809 // END This section of the file is automatically generated. Do not edit --------------
12810
12811
12812 // ============================================================================
12813 // Floating Point Arithmetic Instructions
12814
12815 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12816 match(Set dst (AddHF src1 src2));
12817 format %{ "faddh $dst, $src1, $src2" %}
12818 ins_encode %{
12819 __ faddh($dst$$FloatRegister,
12820 $src1$$FloatRegister,
12821 $src2$$FloatRegister);
12822 %}
12823 ins_pipe(fp_dop_reg_reg_s);
12824 %}
12825
12826 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12827 match(Set dst (AddF src1 src2));
12828
12829 ins_cost(INSN_COST * 5);
12830 format %{ "fadds $dst, $src1, $src2" %}
12831
12832 ins_encode %{
12833 __ fadds(as_FloatRegister($dst$$reg),
12834 as_FloatRegister($src1$$reg),
12835 as_FloatRegister($src2$$reg));
12836 %}
12837
12838 ins_pipe(fp_dop_reg_reg_s);
12839 %}
12840
12841 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12842 match(Set dst (AddD src1 src2));
12843
12844 ins_cost(INSN_COST * 5);
12845 format %{ "faddd $dst, $src1, $src2" %}
12846
12847 ins_encode %{
12848 __ faddd(as_FloatRegister($dst$$reg),
12849 as_FloatRegister($src1$$reg),
12850 as_FloatRegister($src2$$reg));
12851 %}
12852
12853 ins_pipe(fp_dop_reg_reg_d);
12854 %}
12855
12856 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12857 match(Set dst (SubHF src1 src2));
12858 format %{ "fsubh $dst, $src1, $src2" %}
12859 ins_encode %{
12860 __ fsubh($dst$$FloatRegister,
12861 $src1$$FloatRegister,
12862 $src2$$FloatRegister);
12863 %}
12864 ins_pipe(fp_dop_reg_reg_s);
12865 %}
12866
12867 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12868 match(Set dst (SubF src1 src2));
12869
12870 ins_cost(INSN_COST * 5);
12871 format %{ "fsubs $dst, $src1, $src2" %}
12872
12873 ins_encode %{
12874 __ fsubs(as_FloatRegister($dst$$reg),
12875 as_FloatRegister($src1$$reg),
12876 as_FloatRegister($src2$$reg));
12877 %}
12878
12879 ins_pipe(fp_dop_reg_reg_s);
12880 %}
12881
12882 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12883 match(Set dst (SubD src1 src2));
12884
12885 ins_cost(INSN_COST * 5);
12886 format %{ "fsubd $dst, $src1, $src2" %}
12887
12888 ins_encode %{
12889 __ fsubd(as_FloatRegister($dst$$reg),
12890 as_FloatRegister($src1$$reg),
12891 as_FloatRegister($src2$$reg));
12892 %}
12893
12894 ins_pipe(fp_dop_reg_reg_d);
12895 %}
12896
12897 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12898 match(Set dst (MulHF src1 src2));
12899 format %{ "fmulh $dst, $src1, $src2" %}
12900 ins_encode %{
12901 __ fmulh($dst$$FloatRegister,
12902 $src1$$FloatRegister,
12903 $src2$$FloatRegister);
12904 %}
12905 ins_pipe(fp_dop_reg_reg_s);
12906 %}
12907
12908 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12909 match(Set dst (MulF src1 src2));
12910
12911 ins_cost(INSN_COST * 6);
12912 format %{ "fmuls $dst, $src1, $src2" %}
12913
12914 ins_encode %{
12915 __ fmuls(as_FloatRegister($dst$$reg),
12916 as_FloatRegister($src1$$reg),
12917 as_FloatRegister($src2$$reg));
12918 %}
12919
12920 ins_pipe(fp_dop_reg_reg_s);
12921 %}
12922
12923 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12924 match(Set dst (MulD src1 src2));
12925
12926 ins_cost(INSN_COST * 6);
12927 format %{ "fmuld $dst, $src1, $src2" %}
12928
12929 ins_encode %{
12930 __ fmuld(as_FloatRegister($dst$$reg),
12931 as_FloatRegister($src1$$reg),
12932 as_FloatRegister($src2$$reg));
12933 %}
12934
12935 ins_pipe(fp_dop_reg_reg_d);
12936 %}
12937
12938 // src1 * src2 + src3 (half-precision float)
12939 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12940 match(Set dst (FmaHF src3 (Binary src1 src2)));
12941 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12942 ins_encode %{
12943 assert(UseFMA, "Needs FMA instructions support.");
12944 __ fmaddh($dst$$FloatRegister,
12945 $src1$$FloatRegister,
12946 $src2$$FloatRegister,
12947 $src3$$FloatRegister);
12948 %}
12949 ins_pipe(pipe_class_default);
12950 %}
12951
12952 // src1 * src2 + src3
12953 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12954 match(Set dst (FmaF src3 (Binary src1 src2)));
12955
12956 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12957
12958 ins_encode %{
12959 assert(UseFMA, "Needs FMA instructions support.");
12960 __ fmadds(as_FloatRegister($dst$$reg),
12961 as_FloatRegister($src1$$reg),
12962 as_FloatRegister($src2$$reg),
12963 as_FloatRegister($src3$$reg));
12964 %}
12965
12966 ins_pipe(pipe_class_default);
12967 %}
12968
12969 // src1 * src2 + src3
12970 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12971 match(Set dst (FmaD src3 (Binary src1 src2)));
12972
12973 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12974
12975 ins_encode %{
12976 assert(UseFMA, "Needs FMA instructions support.");
12977 __ fmaddd(as_FloatRegister($dst$$reg),
12978 as_FloatRegister($src1$$reg),
12979 as_FloatRegister($src2$$reg),
12980 as_FloatRegister($src3$$reg));
12981 %}
12982
12983 ins_pipe(pipe_class_default);
12984 %}
12985
12986 // src1 * (-src2) + src3
12987 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12988 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12989 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12990
12991 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12992
12993 ins_encode %{
12994 assert(UseFMA, "Needs FMA instructions support.");
12995 __ fmsubs(as_FloatRegister($dst$$reg),
12996 as_FloatRegister($src1$$reg),
12997 as_FloatRegister($src2$$reg),
12998 as_FloatRegister($src3$$reg));
12999 %}
13000
13001 ins_pipe(pipe_class_default);
13002 %}
13003
13004 // src1 * (-src2) + src3
13005 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13006 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13007 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13008
13009 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13010
13011 ins_encode %{
13012 assert(UseFMA, "Needs FMA instructions support.");
13013 __ fmsubd(as_FloatRegister($dst$$reg),
13014 as_FloatRegister($src1$$reg),
13015 as_FloatRegister($src2$$reg),
13016 as_FloatRegister($src3$$reg));
13017 %}
13018
13019 ins_pipe(pipe_class_default);
13020 %}
13021
13022 // src1 * (-src2) - src3
13023 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13024 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13025 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13026
13027 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13028
13029 ins_encode %{
13030 assert(UseFMA, "Needs FMA instructions support.");
13031 __ fnmadds(as_FloatRegister($dst$$reg),
13032 as_FloatRegister($src1$$reg),
13033 as_FloatRegister($src2$$reg),
13034 as_FloatRegister($src3$$reg));
13035 %}
13036
13037 ins_pipe(pipe_class_default);
13038 %}
13039
13040 // src1 * (-src2) - src3
13041 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13042 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13043 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13044
13045 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13046
13047 ins_encode %{
13048 assert(UseFMA, "Needs FMA instructions support.");
13049 __ fnmaddd(as_FloatRegister($dst$$reg),
13050 as_FloatRegister($src1$$reg),
13051 as_FloatRegister($src2$$reg),
13052 as_FloatRegister($src3$$reg));
13053 %}
13054
13055 ins_pipe(pipe_class_default);
13056 %}
13057
13058 // src1 * src2 - src3
13059 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13060 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13061
13062 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13063
13064 ins_encode %{
13065 assert(UseFMA, "Needs FMA instructions support.");
13066 __ fnmsubs(as_FloatRegister($dst$$reg),
13067 as_FloatRegister($src1$$reg),
13068 as_FloatRegister($src2$$reg),
13069 as_FloatRegister($src3$$reg));
13070 %}
13071
13072 ins_pipe(pipe_class_default);
13073 %}
13074
13075 // src1 * src2 - src3
13076 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13077 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13078
13079 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13080
13081 ins_encode %{
13082 assert(UseFMA, "Needs FMA instructions support.");
13083 // n.b. insn name should be fnmsubd
13084 __ fnmsub(as_FloatRegister($dst$$reg),
13085 as_FloatRegister($src1$$reg),
13086 as_FloatRegister($src2$$reg),
13087 as_FloatRegister($src3$$reg));
13088 %}
13089
13090 ins_pipe(pipe_class_default);
13091 %}
13092
13093 // Math.max(HH)H (half-precision float)
13094 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13095 match(Set dst (MaxHF src1 src2));
13096 format %{ "fmaxh $dst, $src1, $src2" %}
13097 ins_encode %{
13098 __ fmaxh($dst$$FloatRegister,
13099 $src1$$FloatRegister,
13100 $src2$$FloatRegister);
13101 %}
13102 ins_pipe(fp_dop_reg_reg_s);
13103 %}
13104
13105 // Math.min(HH)H (half-precision float)
13106 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13107 match(Set dst (MinHF src1 src2));
13108 format %{ "fminh $dst, $src1, $src2" %}
13109 ins_encode %{
13110 __ fminh($dst$$FloatRegister,
13111 $src1$$FloatRegister,
13112 $src2$$FloatRegister);
13113 %}
13114 ins_pipe(fp_dop_reg_reg_s);
13115 %}
13116
13117 // Math.max(FF)F
13118 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13119 match(Set dst (MaxF src1 src2));
13120
13121 format %{ "fmaxs $dst, $src1, $src2" %}
13122 ins_encode %{
13123 __ fmaxs(as_FloatRegister($dst$$reg),
13124 as_FloatRegister($src1$$reg),
13125 as_FloatRegister($src2$$reg));
13126 %}
13127
13128 ins_pipe(fp_dop_reg_reg_s);
13129 %}
13130
13131 // Math.min(FF)F
13132 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13133 match(Set dst (MinF src1 src2));
13134
13135 format %{ "fmins $dst, $src1, $src2" %}
13136 ins_encode %{
13137 __ fmins(as_FloatRegister($dst$$reg),
13138 as_FloatRegister($src1$$reg),
13139 as_FloatRegister($src2$$reg));
13140 %}
13141
13142 ins_pipe(fp_dop_reg_reg_s);
13143 %}
13144
13145 // Math.max(DD)D
13146 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13147 match(Set dst (MaxD src1 src2));
13148
13149 format %{ "fmaxd $dst, $src1, $src2" %}
13150 ins_encode %{
13151 __ fmaxd(as_FloatRegister($dst$$reg),
13152 as_FloatRegister($src1$$reg),
13153 as_FloatRegister($src2$$reg));
13154 %}
13155
13156 ins_pipe(fp_dop_reg_reg_d);
13157 %}
13158
13159 // Math.min(DD)D
13160 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13161 match(Set dst (MinD src1 src2));
13162
13163 format %{ "fmind $dst, $src1, $src2" %}
13164 ins_encode %{
13165 __ fmind(as_FloatRegister($dst$$reg),
13166 as_FloatRegister($src1$$reg),
13167 as_FloatRegister($src2$$reg));
13168 %}
13169
13170 ins_pipe(fp_dop_reg_reg_d);
13171 %}
13172
13173 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13174 match(Set dst (DivHF src1 src2));
13175 format %{ "fdivh $dst, $src1, $src2" %}
13176 ins_encode %{
13177 __ fdivh($dst$$FloatRegister,
13178 $src1$$FloatRegister,
13179 $src2$$FloatRegister);
13180 %}
13181 ins_pipe(fp_div_s);
13182 %}
13183
13184 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13185 match(Set dst (DivF src1 src2));
13186
13187 ins_cost(INSN_COST * 18);
13188 format %{ "fdivs $dst, $src1, $src2" %}
13189
13190 ins_encode %{
13191 __ fdivs(as_FloatRegister($dst$$reg),
13192 as_FloatRegister($src1$$reg),
13193 as_FloatRegister($src2$$reg));
13194 %}
13195
13196 ins_pipe(fp_div_s);
13197 %}
13198
13199 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13200 match(Set dst (DivD src1 src2));
13201
13202 ins_cost(INSN_COST * 32);
13203 format %{ "fdivd $dst, $src1, $src2" %}
13204
13205 ins_encode %{
13206 __ fdivd(as_FloatRegister($dst$$reg),
13207 as_FloatRegister($src1$$reg),
13208 as_FloatRegister($src2$$reg));
13209 %}
13210
13211 ins_pipe(fp_div_d);
13212 %}
13213
13214 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13215 match(Set dst (NegF src));
13216
13217 ins_cost(INSN_COST * 3);
13218 format %{ "fneg $dst, $src" %}
13219
13220 ins_encode %{
13221 __ fnegs(as_FloatRegister($dst$$reg),
13222 as_FloatRegister($src$$reg));
13223 %}
13224
13225 ins_pipe(fp_uop_s);
13226 %}
13227
13228 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13229 match(Set dst (NegD src));
13230
13231 ins_cost(INSN_COST * 3);
13232 format %{ "fnegd $dst, $src" %}
13233
13234 ins_encode %{
13235 __ fnegd(as_FloatRegister($dst$$reg),
13236 as_FloatRegister($src$$reg));
13237 %}
13238
13239 ins_pipe(fp_uop_d);
13240 %}
13241
13242 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13243 %{
13244 match(Set dst (AbsI src));
13245
13246 effect(KILL cr);
13247 ins_cost(INSN_COST * 2);
13248 format %{ "cmpw $src, zr\n\t"
13249 "cnegw $dst, $src, Assembler::LT\t# int abs"
13250 %}
13251
13252 ins_encode %{
13253 __ cmpw(as_Register($src$$reg), zr);
13254 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13255 %}
13256 ins_pipe(pipe_class_default);
13257 %}
13258
13259 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13260 %{
13261 match(Set dst (AbsL src));
13262
13263 effect(KILL cr);
13264 ins_cost(INSN_COST * 2);
13265 format %{ "cmp $src, zr\n\t"
13266 "cneg $dst, $src, Assembler::LT\t# long abs"
13267 %}
13268
13269 ins_encode %{
13270 __ cmp(as_Register($src$$reg), zr);
13271 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13272 %}
13273 ins_pipe(pipe_class_default);
13274 %}
13275
13276 instruct absF_reg(vRegF dst, vRegF src) %{
13277 match(Set dst (AbsF src));
13278
13279 ins_cost(INSN_COST * 3);
13280 format %{ "fabss $dst, $src" %}
13281 ins_encode %{
13282 __ fabss(as_FloatRegister($dst$$reg),
13283 as_FloatRegister($src$$reg));
13284 %}
13285
13286 ins_pipe(fp_uop_s);
13287 %}
13288
13289 instruct absD_reg(vRegD dst, vRegD src) %{
13290 match(Set dst (AbsD src));
13291
13292 ins_cost(INSN_COST * 3);
13293 format %{ "fabsd $dst, $src" %}
13294 ins_encode %{
13295 __ fabsd(as_FloatRegister($dst$$reg),
13296 as_FloatRegister($src$$reg));
13297 %}
13298
13299 ins_pipe(fp_uop_d);
13300 %}
13301
13302 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13303 match(Set dst (AbsF (SubF src1 src2)));
13304
13305 ins_cost(INSN_COST * 3);
13306 format %{ "fabds $dst, $src1, $src2" %}
13307 ins_encode %{
13308 __ fabds(as_FloatRegister($dst$$reg),
13309 as_FloatRegister($src1$$reg),
13310 as_FloatRegister($src2$$reg));
13311 %}
13312
13313 ins_pipe(fp_uop_s);
13314 %}
13315
13316 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13317 match(Set dst (AbsD (SubD src1 src2)));
13318
13319 ins_cost(INSN_COST * 3);
13320 format %{ "fabdd $dst, $src1, $src2" %}
13321 ins_encode %{
13322 __ fabdd(as_FloatRegister($dst$$reg),
13323 as_FloatRegister($src1$$reg),
13324 as_FloatRegister($src2$$reg));
13325 %}
13326
13327 ins_pipe(fp_uop_d);
13328 %}
13329
13330 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13331 match(Set dst (SqrtD src));
13332
13333 ins_cost(INSN_COST * 50);
13334 format %{ "fsqrtd $dst, $src" %}
13335 ins_encode %{
13336 __ fsqrtd(as_FloatRegister($dst$$reg),
13337 as_FloatRegister($src$$reg));
13338 %}
13339
13340 ins_pipe(fp_div_s);
13341 %}
13342
13343 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13344 match(Set dst (SqrtF src));
13345
13346 ins_cost(INSN_COST * 50);
13347 format %{ "fsqrts $dst, $src" %}
13348 ins_encode %{
13349 __ fsqrts(as_FloatRegister($dst$$reg),
13350 as_FloatRegister($src$$reg));
13351 %}
13352
13353 ins_pipe(fp_div_d);
13354 %}
13355
13356 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13357 match(Set dst (SqrtHF src));
13358 format %{ "fsqrth $dst, $src" %}
13359 ins_encode %{
13360 __ fsqrth($dst$$FloatRegister,
13361 $src$$FloatRegister);
13362 %}
13363 ins_pipe(fp_div_s);
13364 %}
13365
13366 // Math.rint, floor, ceil
13367 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13368 match(Set dst (RoundDoubleMode src rmode));
13369 format %{ "frint $dst, $src, $rmode" %}
13370 ins_encode %{
13371 switch ($rmode$$constant) {
13372 case RoundDoubleModeNode::rmode_rint:
13373 __ frintnd(as_FloatRegister($dst$$reg),
13374 as_FloatRegister($src$$reg));
13375 break;
13376 case RoundDoubleModeNode::rmode_floor:
13377 __ frintmd(as_FloatRegister($dst$$reg),
13378 as_FloatRegister($src$$reg));
13379 break;
13380 case RoundDoubleModeNode::rmode_ceil:
13381 __ frintpd(as_FloatRegister($dst$$reg),
13382 as_FloatRegister($src$$reg));
13383 break;
13384 }
13385 %}
13386 ins_pipe(fp_uop_d);
13387 %}
13388
13389 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13390 match(Set dst (CopySignD src1 (Binary src2 zero)));
13391 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13392 format %{ "CopySignD $dst $src1 $src2" %}
13393 ins_encode %{
13394 FloatRegister dst = as_FloatRegister($dst$$reg),
13395 src1 = as_FloatRegister($src1$$reg),
13396 src2 = as_FloatRegister($src2$$reg),
13397 zero = as_FloatRegister($zero$$reg);
13398 __ fnegd(dst, zero);
13399 __ bsl(dst, __ T8B, src2, src1);
13400 %}
13401 ins_pipe(fp_uop_d);
13402 %}
13403
13404 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13405 match(Set dst (CopySignF src1 src2));
13406 effect(TEMP_DEF dst, USE src1, USE src2);
13407 format %{ "CopySignF $dst $src1 $src2" %}
13408 ins_encode %{
13409 FloatRegister dst = as_FloatRegister($dst$$reg),
13410 src1 = as_FloatRegister($src1$$reg),
13411 src2 = as_FloatRegister($src2$$reg);
13412 __ movi(dst, __ T2S, 0x80, 24);
13413 __ bsl(dst, __ T8B, src2, src1);
13414 %}
13415 ins_pipe(fp_uop_d);
13416 %}
13417
13418 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13419 match(Set dst (SignumD src (Binary zero one)));
13420 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13421 format %{ "signumD $dst, $src" %}
13422 ins_encode %{
13423 FloatRegister src = as_FloatRegister($src$$reg),
13424 dst = as_FloatRegister($dst$$reg),
13425 zero = as_FloatRegister($zero$$reg),
13426 one = as_FloatRegister($one$$reg);
13427 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13428 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13429 // Bit selection instruction gets bit from "one" for each enabled bit in
13430 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13431 // NaN the whole "src" will be copied because "dst" is zero. For all other
13432 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13433 // from "src", and all other bits are copied from 1.0.
13434 __ bsl(dst, __ T8B, one, src);
13435 %}
13436 ins_pipe(fp_uop_d);
13437 %}
13438
13439 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13440 match(Set dst (SignumF src (Binary zero one)));
13441 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13442 format %{ "signumF $dst, $src" %}
13443 ins_encode %{
13444 FloatRegister src = as_FloatRegister($src$$reg),
13445 dst = as_FloatRegister($dst$$reg),
13446 zero = as_FloatRegister($zero$$reg),
13447 one = as_FloatRegister($one$$reg);
13448 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13449 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13450 // Bit selection instruction gets bit from "one" for each enabled bit in
13451 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13452 // NaN the whole "src" will be copied because "dst" is zero. For all other
13453 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13454 // from "src", and all other bits are copied from 1.0.
13455 __ bsl(dst, __ T8B, one, src);
13456 %}
13457 ins_pipe(fp_uop_d);
13458 %}
13459
13460 instruct onspinwait() %{
13461 match(OnSpinWait);
13462 ins_cost(INSN_COST);
13463
13464 format %{ "onspinwait" %}
13465
13466 ins_encode %{
13467 __ spin_wait();
13468 %}
13469 ins_pipe(pipe_class_empty);
13470 %}
13471
13472 // ============================================================================
13473 // Logical Instructions
13474
13475 // Integer Logical Instructions
13476
13477 // And Instructions
13478
13479
13480 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13481 match(Set dst (AndI src1 src2));
13482
13483 format %{ "andw $dst, $src1, $src2\t# int" %}
13484
13485 ins_cost(INSN_COST);
13486 ins_encode %{
13487 __ andw(as_Register($dst$$reg),
13488 as_Register($src1$$reg),
13489 as_Register($src2$$reg));
13490 %}
13491
13492 ins_pipe(ialu_reg_reg);
13493 %}
13494
13495 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13496 match(Set dst (AndI src1 src2));
13497
13498 format %{ "andsw $dst, $src1, $src2\t# int" %}
13499
13500 ins_cost(INSN_COST);
13501 ins_encode %{
13502 __ andw(as_Register($dst$$reg),
13503 as_Register($src1$$reg),
13504 (uint64_t)($src2$$constant));
13505 %}
13506
13507 ins_pipe(ialu_reg_imm);
13508 %}
13509
13510 // Or Instructions
13511
13512 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13513 match(Set dst (OrI src1 src2));
13514
13515 format %{ "orrw $dst, $src1, $src2\t# int" %}
13516
13517 ins_cost(INSN_COST);
13518 ins_encode %{
13519 __ orrw(as_Register($dst$$reg),
13520 as_Register($src1$$reg),
13521 as_Register($src2$$reg));
13522 %}
13523
13524 ins_pipe(ialu_reg_reg);
13525 %}
13526
13527 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13528 match(Set dst (OrI src1 src2));
13529
13530 format %{ "orrw $dst, $src1, $src2\t# int" %}
13531
13532 ins_cost(INSN_COST);
13533 ins_encode %{
13534 __ orrw(as_Register($dst$$reg),
13535 as_Register($src1$$reg),
13536 (uint64_t)($src2$$constant));
13537 %}
13538
13539 ins_pipe(ialu_reg_imm);
13540 %}
13541
13542 // Xor Instructions
13543
13544 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13545 match(Set dst (XorI src1 src2));
13546
13547 format %{ "eorw $dst, $src1, $src2\t# int" %}
13548
13549 ins_cost(INSN_COST);
13550 ins_encode %{
13551 __ eorw(as_Register($dst$$reg),
13552 as_Register($src1$$reg),
13553 as_Register($src2$$reg));
13554 %}
13555
13556 ins_pipe(ialu_reg_reg);
13557 %}
13558
13559 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13560 match(Set dst (XorI src1 src2));
13561
13562 format %{ "eorw $dst, $src1, $src2\t# int" %}
13563
13564 ins_cost(INSN_COST);
13565 ins_encode %{
13566 __ eorw(as_Register($dst$$reg),
13567 as_Register($src1$$reg),
13568 (uint64_t)($src2$$constant));
13569 %}
13570
13571 ins_pipe(ialu_reg_imm);
13572 %}
13573
13574 // Long Logical Instructions
13575 // TODO
13576
13577 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13578 match(Set dst (AndL src1 src2));
13579
13580 format %{ "and $dst, $src1, $src2\t# int" %}
13581
13582 ins_cost(INSN_COST);
13583 ins_encode %{
13584 __ andr(as_Register($dst$$reg),
13585 as_Register($src1$$reg),
13586 as_Register($src2$$reg));
13587 %}
13588
13589 ins_pipe(ialu_reg_reg);
13590 %}
13591
13592 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13593 match(Set dst (AndL src1 src2));
13594
13595 format %{ "and $dst, $src1, $src2\t# int" %}
13596
13597 ins_cost(INSN_COST);
13598 ins_encode %{
13599 __ andr(as_Register($dst$$reg),
13600 as_Register($src1$$reg),
13601 (uint64_t)($src2$$constant));
13602 %}
13603
13604 ins_pipe(ialu_reg_imm);
13605 %}
13606
13607 // Or Instructions
13608
13609 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13610 match(Set dst (OrL src1 src2));
13611
13612 format %{ "orr $dst, $src1, $src2\t# int" %}
13613
13614 ins_cost(INSN_COST);
13615 ins_encode %{
13616 __ orr(as_Register($dst$$reg),
13617 as_Register($src1$$reg),
13618 as_Register($src2$$reg));
13619 %}
13620
13621 ins_pipe(ialu_reg_reg);
13622 %}
13623
13624 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13625 match(Set dst (OrL src1 src2));
13626
13627 format %{ "orr $dst, $src1, $src2\t# int" %}
13628
13629 ins_cost(INSN_COST);
13630 ins_encode %{
13631 __ orr(as_Register($dst$$reg),
13632 as_Register($src1$$reg),
13633 (uint64_t)($src2$$constant));
13634 %}
13635
13636 ins_pipe(ialu_reg_imm);
13637 %}
13638
13639 // Xor Instructions
13640
13641 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13642 match(Set dst (XorL src1 src2));
13643
13644 format %{ "eor $dst, $src1, $src2\t# int" %}
13645
13646 ins_cost(INSN_COST);
13647 ins_encode %{
13648 __ eor(as_Register($dst$$reg),
13649 as_Register($src1$$reg),
13650 as_Register($src2$$reg));
13651 %}
13652
13653 ins_pipe(ialu_reg_reg);
13654 %}
13655
13656 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13657 match(Set dst (XorL src1 src2));
13658
13659 ins_cost(INSN_COST);
13660 format %{ "eor $dst, $src1, $src2\t# int" %}
13661
13662 ins_encode %{
13663 __ eor(as_Register($dst$$reg),
13664 as_Register($src1$$reg),
13665 (uint64_t)($src2$$constant));
13666 %}
13667
13668 ins_pipe(ialu_reg_imm);
13669 %}
13670
13671 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13672 %{
13673 match(Set dst (ConvI2L src));
13674
13675 ins_cost(INSN_COST);
13676 format %{ "sxtw $dst, $src\t# i2l" %}
13677 ins_encode %{
13678 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13679 %}
13680 ins_pipe(ialu_reg_shift);
13681 %}
13682
13683 // this pattern occurs in bigmath arithmetic
13684 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13685 %{
13686 match(Set dst (AndL (ConvI2L src) mask));
13687
13688 ins_cost(INSN_COST);
13689 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13690 ins_encode %{
13691 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13692 %}
13693
13694 ins_pipe(ialu_reg_shift);
13695 %}
13696
13697 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13698 match(Set dst (ConvL2I src));
13699
13700 ins_cost(INSN_COST);
13701 format %{ "movw $dst, $src \t// l2i" %}
13702
13703 ins_encode %{
13704 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13705 %}
13706
13707 ins_pipe(ialu_reg);
13708 %}
13709
13710 instruct convD2F_reg(vRegF dst, vRegD src) %{
13711 match(Set dst (ConvD2F src));
13712
13713 ins_cost(INSN_COST * 5);
13714 format %{ "fcvtd $dst, $src \t// d2f" %}
13715
13716 ins_encode %{
13717 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13718 %}
13719
13720 ins_pipe(fp_d2f);
13721 %}
13722
13723 instruct convF2D_reg(vRegD dst, vRegF src) %{
13724 match(Set dst (ConvF2D src));
13725
13726 ins_cost(INSN_COST * 5);
13727 format %{ "fcvts $dst, $src \t// f2d" %}
13728
13729 ins_encode %{
13730 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13731 %}
13732
13733 ins_pipe(fp_f2d);
13734 %}
13735
13736 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13737 match(Set dst (ConvF2I src));
13738
13739 ins_cost(INSN_COST * 5);
13740 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13741
13742 ins_encode %{
13743 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13744 %}
13745
13746 ins_pipe(fp_f2i);
13747 %}
13748
13749 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13750 match(Set dst (ConvF2L src));
13751
13752 ins_cost(INSN_COST * 5);
13753 format %{ "fcvtzs $dst, $src \t// f2l" %}
13754
13755 ins_encode %{
13756 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13757 %}
13758
13759 ins_pipe(fp_f2l);
13760 %}
13761
13762 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13763 match(Set dst (ConvF2HF src));
13764 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13765 "smov $dst, $tmp\t# move result from $tmp to $dst"
13766 %}
13767 effect(TEMP tmp);
13768 ins_encode %{
13769 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13770 %}
13771 ins_pipe(pipe_slow);
13772 %}
13773
13774 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13775 match(Set dst (ConvHF2F src));
13776 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13777 "fcvt $dst, $tmp\t# convert half to single precision"
13778 %}
13779 effect(TEMP tmp);
13780 ins_encode %{
13781 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13782 %}
13783 ins_pipe(pipe_slow);
13784 %}
13785
13786 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13787 match(Set dst (ConvI2F src));
13788
13789 ins_cost(INSN_COST * 5);
13790 format %{ "scvtfws $dst, $src \t// i2f" %}
13791
13792 ins_encode %{
13793 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13794 %}
13795
13796 ins_pipe(fp_i2f);
13797 %}
13798
13799 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13800 match(Set dst (ConvL2F src));
13801
13802 ins_cost(INSN_COST * 5);
13803 format %{ "scvtfs $dst, $src \t// l2f" %}
13804
13805 ins_encode %{
13806 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13807 %}
13808
13809 ins_pipe(fp_l2f);
13810 %}
13811
13812 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13813 match(Set dst (ConvD2I src));
13814
13815 ins_cost(INSN_COST * 5);
13816 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13817
13818 ins_encode %{
13819 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13820 %}
13821
13822 ins_pipe(fp_d2i);
13823 %}
13824
13825 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13826 match(Set dst (ConvD2L src));
13827
13828 ins_cost(INSN_COST * 5);
13829 format %{ "fcvtzd $dst, $src \t// d2l" %}
13830
13831 ins_encode %{
13832 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13833 %}
13834
13835 ins_pipe(fp_d2l);
13836 %}
13837
13838 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13839 match(Set dst (ConvI2D src));
13840
13841 ins_cost(INSN_COST * 5);
13842 format %{ "scvtfwd $dst, $src \t// i2d" %}
13843
13844 ins_encode %{
13845 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13846 %}
13847
13848 ins_pipe(fp_i2d);
13849 %}
13850
13851 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13852 match(Set dst (ConvL2D src));
13853
13854 ins_cost(INSN_COST * 5);
13855 format %{ "scvtfd $dst, $src \t// l2d" %}
13856
13857 ins_encode %{
13858 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13859 %}
13860
13861 ins_pipe(fp_l2d);
13862 %}
13863
13864 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13865 %{
13866 match(Set dst (RoundD src));
13867 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13868 format %{ "java_round_double $dst,$src"%}
13869 ins_encode %{
13870 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13871 as_FloatRegister($ftmp$$reg));
13872 %}
13873 ins_pipe(pipe_slow);
13874 %}
13875
13876 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13877 %{
13878 match(Set dst (RoundF src));
13879 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13880 format %{ "java_round_float $dst,$src"%}
13881 ins_encode %{
13882 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13883 as_FloatRegister($ftmp$$reg));
13884 %}
13885 ins_pipe(pipe_slow);
13886 %}
13887
13888 // stack <-> reg and reg <-> reg shuffles with no conversion
13889
13890 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13891
13892 match(Set dst (MoveF2I src));
13893
13894 effect(DEF dst, USE src);
13895
13896 ins_cost(4 * INSN_COST);
13897
13898 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13899
13900 ins_encode %{
13901 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13902 %}
13903
13904 ins_pipe(iload_reg_reg);
13905
13906 %}
13907
13908 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13909
13910 match(Set dst (MoveI2F src));
13911
13912 effect(DEF dst, USE src);
13913
13914 ins_cost(4 * INSN_COST);
13915
13916 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13917
13918 ins_encode %{
13919 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13920 %}
13921
13922 ins_pipe(pipe_class_memory);
13923
13924 %}
13925
13926 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13927
13928 match(Set dst (MoveD2L src));
13929
13930 effect(DEF dst, USE src);
13931
13932 ins_cost(4 * INSN_COST);
13933
13934 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13935
13936 ins_encode %{
13937 __ ldr($dst$$Register, Address(sp, $src$$disp));
13938 %}
13939
13940 ins_pipe(iload_reg_reg);
13941
13942 %}
13943
13944 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13945
13946 match(Set dst (MoveL2D src));
13947
13948 effect(DEF dst, USE src);
13949
13950 ins_cost(4 * INSN_COST);
13951
13952 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13953
13954 ins_encode %{
13955 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13956 %}
13957
13958 ins_pipe(pipe_class_memory);
13959
13960 %}
13961
13962 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13963
13964 match(Set dst (MoveF2I src));
13965
13966 effect(DEF dst, USE src);
13967
13968 ins_cost(INSN_COST);
13969
13970 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13971
13972 ins_encode %{
13973 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13974 %}
13975
13976 ins_pipe(pipe_class_memory);
13977
13978 %}
13979
13980 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13981
13982 match(Set dst (MoveI2F src));
13983
13984 effect(DEF dst, USE src);
13985
13986 ins_cost(INSN_COST);
13987
13988 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13989
13990 ins_encode %{
13991 __ strw($src$$Register, Address(sp, $dst$$disp));
13992 %}
13993
13994 ins_pipe(istore_reg_reg);
13995
13996 %}
13997
13998 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13999
14000 match(Set dst (MoveD2L src));
14001
14002 effect(DEF dst, USE src);
14003
14004 ins_cost(INSN_COST);
14005
14006 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14007
14008 ins_encode %{
14009 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14010 %}
14011
14012 ins_pipe(pipe_class_memory);
14013
14014 %}
14015
14016 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14017
14018 match(Set dst (MoveL2D src));
14019
14020 effect(DEF dst, USE src);
14021
14022 ins_cost(INSN_COST);
14023
14024 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14025
14026 ins_encode %{
14027 __ str($src$$Register, Address(sp, $dst$$disp));
14028 %}
14029
14030 ins_pipe(istore_reg_reg);
14031
14032 %}
14033
14034 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14035
14036 match(Set dst (MoveF2I src));
14037
14038 effect(DEF dst, USE src);
14039
14040 ins_cost(INSN_COST);
14041
14042 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14043
14044 ins_encode %{
14045 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14046 %}
14047
14048 ins_pipe(fp_f2i);
14049
14050 %}
14051
14052 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14053
14054 match(Set dst (MoveI2F src));
14055
14056 effect(DEF dst, USE src);
14057
14058 ins_cost(INSN_COST);
14059
14060 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14061
14062 ins_encode %{
14063 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14064 %}
14065
14066 ins_pipe(fp_i2f);
14067
14068 %}
14069
14070 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14071
14072 match(Set dst (MoveD2L src));
14073
14074 effect(DEF dst, USE src);
14075
14076 ins_cost(INSN_COST);
14077
14078 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14079
14080 ins_encode %{
14081 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14082 %}
14083
14084 ins_pipe(fp_d2l);
14085
14086 %}
14087
14088 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14089
14090 match(Set dst (MoveL2D src));
14091
14092 effect(DEF dst, USE src);
14093
14094 ins_cost(INSN_COST);
14095
14096 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14097
14098 ins_encode %{
14099 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14100 %}
14101
14102 ins_pipe(fp_l2d);
14103
14104 %}
14105
14106 // ============================================================================
14107 // clearing of an array
14108
14109 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14110 %{
14111 match(Set dummy (ClearArray cnt base));
14112 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14113
14114 ins_cost(4 * INSN_COST);
14115 format %{ "ClearArray $cnt, $base" %}
14116
14117 ins_encode %{
14118 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14119 if (tpc == nullptr) {
14120 ciEnv::current()->record_failure("CodeCache is full");
14121 return;
14122 }
14123 %}
14124
14125 ins_pipe(pipe_class_memory);
14126 %}
14127
14128 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14129 %{
14130 predicate((uint64_t)n->in(2)->get_long()
14131 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14132 match(Set dummy (ClearArray cnt base));
14133 effect(TEMP temp, USE_KILL base, KILL cr);
14134
14135 ins_cost(4 * INSN_COST);
14136 format %{ "ClearArray $cnt, $base" %}
14137
14138 ins_encode %{
14139 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14140 if (tpc == nullptr) {
14141 ciEnv::current()->record_failure("CodeCache is full");
14142 return;
14143 }
14144 %}
14145
14146 ins_pipe(pipe_class_memory);
14147 %}
14148
14149 // ============================================================================
14150 // Overflow Math Instructions
14151
14152 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14153 %{
14154 match(Set cr (OverflowAddI op1 op2));
14155
14156 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14157 ins_cost(INSN_COST);
14158 ins_encode %{
14159 __ cmnw($op1$$Register, $op2$$Register);
14160 %}
14161
14162 ins_pipe(icmp_reg_reg);
14163 %}
14164
14165 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14166 %{
14167 match(Set cr (OverflowAddI op1 op2));
14168
14169 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14170 ins_cost(INSN_COST);
14171 ins_encode %{
14172 __ cmnw($op1$$Register, $op2$$constant);
14173 %}
14174
14175 ins_pipe(icmp_reg_imm);
14176 %}
14177
14178 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14179 %{
14180 match(Set cr (OverflowAddL op1 op2));
14181
14182 format %{ "cmn $op1, $op2\t# overflow check long" %}
14183 ins_cost(INSN_COST);
14184 ins_encode %{
14185 __ cmn($op1$$Register, $op2$$Register);
14186 %}
14187
14188 ins_pipe(icmp_reg_reg);
14189 %}
14190
14191 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14192 %{
14193 match(Set cr (OverflowAddL op1 op2));
14194
14195 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14196 ins_cost(INSN_COST);
14197 ins_encode %{
14198 __ adds(zr, $op1$$Register, $op2$$constant);
14199 %}
14200
14201 ins_pipe(icmp_reg_imm);
14202 %}
14203
14204 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14205 %{
14206 match(Set cr (OverflowSubI op1 op2));
14207
14208 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14209 ins_cost(INSN_COST);
14210 ins_encode %{
14211 __ cmpw($op1$$Register, $op2$$Register);
14212 %}
14213
14214 ins_pipe(icmp_reg_reg);
14215 %}
14216
14217 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14218 %{
14219 match(Set cr (OverflowSubI op1 op2));
14220
14221 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14222 ins_cost(INSN_COST);
14223 ins_encode %{
14224 __ cmpw($op1$$Register, $op2$$constant);
14225 %}
14226
14227 ins_pipe(icmp_reg_imm);
14228 %}
14229
14230 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14231 %{
14232 match(Set cr (OverflowSubL op1 op2));
14233
14234 format %{ "cmp $op1, $op2\t# overflow check long" %}
14235 ins_cost(INSN_COST);
14236 ins_encode %{
14237 __ cmp($op1$$Register, $op2$$Register);
14238 %}
14239
14240 ins_pipe(icmp_reg_reg);
14241 %}
14242
14243 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14244 %{
14245 match(Set cr (OverflowSubL op1 op2));
14246
14247 format %{ "cmp $op1, $op2\t# overflow check long" %}
14248 ins_cost(INSN_COST);
14249 ins_encode %{
14250 __ subs(zr, $op1$$Register, $op2$$constant);
14251 %}
14252
14253 ins_pipe(icmp_reg_imm);
14254 %}
14255
14256 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14257 %{
14258 match(Set cr (OverflowSubI zero op1));
14259
14260 format %{ "cmpw zr, $op1\t# overflow check int" %}
14261 ins_cost(INSN_COST);
14262 ins_encode %{
14263 __ cmpw(zr, $op1$$Register);
14264 %}
14265
14266 ins_pipe(icmp_reg_imm);
14267 %}
14268
14269 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14270 %{
14271 match(Set cr (OverflowSubL zero op1));
14272
14273 format %{ "cmp zr, $op1\t# overflow check long" %}
14274 ins_cost(INSN_COST);
14275 ins_encode %{
14276 __ cmp(zr, $op1$$Register);
14277 %}
14278
14279 ins_pipe(icmp_reg_imm);
14280 %}
14281
14282 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14283 %{
14284 match(Set cr (OverflowMulI op1 op2));
14285
14286 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14287 "cmp rscratch1, rscratch1, sxtw\n\t"
14288 "movw rscratch1, #0x80000000\n\t"
14289 "cselw rscratch1, rscratch1, zr, NE\n\t"
14290 "cmpw rscratch1, #1" %}
14291 ins_cost(5 * INSN_COST);
14292 ins_encode %{
14293 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14294 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14295 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14296 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14297 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14298 %}
14299
14300 ins_pipe(pipe_slow);
14301 %}
14302
14303 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14304 %{
14305 match(If cmp (OverflowMulI op1 op2));
14306 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14307 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14308 effect(USE labl, KILL cr);
14309
14310 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14311 "cmp rscratch1, rscratch1, sxtw\n\t"
14312 "b$cmp $labl" %}
14313 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14314 ins_encode %{
14315 Label* L = $labl$$label;
14316 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14317 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14318 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14319 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14320 %}
14321
14322 ins_pipe(pipe_serial);
14323 %}
14324
14325 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14326 %{
14327 match(Set cr (OverflowMulL op1 op2));
14328
14329 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14330 "smulh rscratch2, $op1, $op2\n\t"
14331 "cmp rscratch2, rscratch1, ASR #63\n\t"
14332 "movw rscratch1, #0x80000000\n\t"
14333 "cselw rscratch1, rscratch1, zr, NE\n\t"
14334 "cmpw rscratch1, #1" %}
14335 ins_cost(6 * INSN_COST);
14336 ins_encode %{
14337 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14338 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14339 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14340 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14341 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14342 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14343 %}
14344
14345 ins_pipe(pipe_slow);
14346 %}
14347
14348 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14349 %{
14350 match(If cmp (OverflowMulL op1 op2));
14351 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14352 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14353 effect(USE labl, KILL cr);
14354
14355 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14356 "smulh rscratch2, $op1, $op2\n\t"
14357 "cmp rscratch2, rscratch1, ASR #63\n\t"
14358 "b$cmp $labl" %}
14359 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14360 ins_encode %{
14361 Label* L = $labl$$label;
14362 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14363 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14364 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14365 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14366 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14367 %}
14368
14369 ins_pipe(pipe_serial);
14370 %}
14371
14372 // ============================================================================
14373 // Compare Instructions
14374
14375 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14376 %{
14377 match(Set cr (CmpI op1 op2));
14378
14379 effect(DEF cr, USE op1, USE op2);
14380
14381 ins_cost(INSN_COST);
14382 format %{ "cmpw $op1, $op2" %}
14383
14384 ins_encode(aarch64_enc_cmpw(op1, op2));
14385
14386 ins_pipe(icmp_reg_reg);
14387 %}
14388
14389 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14390 %{
14391 match(Set cr (CmpI op1 zero));
14392
14393 effect(DEF cr, USE op1);
14394
14395 ins_cost(INSN_COST);
14396 format %{ "cmpw $op1, 0" %}
14397
14398 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14399
14400 ins_pipe(icmp_reg_imm);
14401 %}
14402
14403 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14404 %{
14405 match(Set cr (CmpI op1 op2));
14406
14407 effect(DEF cr, USE op1);
14408
14409 ins_cost(INSN_COST);
14410 format %{ "cmpw $op1, $op2" %}
14411
14412 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14413
14414 ins_pipe(icmp_reg_imm);
14415 %}
14416
14417 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14418 %{
14419 match(Set cr (CmpI op1 op2));
14420
14421 effect(DEF cr, USE op1);
14422
14423 ins_cost(INSN_COST * 2);
14424 format %{ "cmpw $op1, $op2" %}
14425
14426 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14427
14428 ins_pipe(icmp_reg_imm);
14429 %}
14430
14431 // Unsigned compare Instructions; really, same as signed compare
14432 // except it should only be used to feed an If or a CMovI which takes a
14433 // cmpOpU.
14434
14435 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14436 %{
14437 match(Set cr (CmpU op1 op2));
14438
14439 effect(DEF cr, USE op1, USE op2);
14440
14441 ins_cost(INSN_COST);
14442 format %{ "cmpw $op1, $op2\t# unsigned" %}
14443
14444 ins_encode(aarch64_enc_cmpw(op1, op2));
14445
14446 ins_pipe(icmp_reg_reg);
14447 %}
14448
14449 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14450 %{
14451 match(Set cr (CmpU op1 zero));
14452
14453 effect(DEF cr, USE op1);
14454
14455 ins_cost(INSN_COST);
14456 format %{ "cmpw $op1, #0\t# unsigned" %}
14457
14458 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14459
14460 ins_pipe(icmp_reg_imm);
14461 %}
14462
14463 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14464 %{
14465 match(Set cr (CmpU op1 op2));
14466
14467 effect(DEF cr, USE op1);
14468
14469 ins_cost(INSN_COST);
14470 format %{ "cmpw $op1, $op2\t# unsigned" %}
14471
14472 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14473
14474 ins_pipe(icmp_reg_imm);
14475 %}
14476
14477 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14478 %{
14479 match(Set cr (CmpU op1 op2));
14480
14481 effect(DEF cr, USE op1);
14482
14483 ins_cost(INSN_COST * 2);
14484 format %{ "cmpw $op1, $op2\t# unsigned" %}
14485
14486 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14487
14488 ins_pipe(icmp_reg_imm);
14489 %}
14490
14491 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14492 %{
14493 match(Set cr (CmpL op1 op2));
14494
14495 effect(DEF cr, USE op1, USE op2);
14496
14497 ins_cost(INSN_COST);
14498 format %{ "cmp $op1, $op2" %}
14499
14500 ins_encode(aarch64_enc_cmp(op1, op2));
14501
14502 ins_pipe(icmp_reg_reg);
14503 %}
14504
14505 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14506 %{
14507 match(Set cr (CmpL op1 zero));
14508
14509 effect(DEF cr, USE op1);
14510
14511 ins_cost(INSN_COST);
14512 format %{ "tst $op1" %}
14513
14514 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14515
14516 ins_pipe(icmp_reg_imm);
14517 %}
14518
14519 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14520 %{
14521 match(Set cr (CmpL op1 op2));
14522
14523 effect(DEF cr, USE op1);
14524
14525 ins_cost(INSN_COST);
14526 format %{ "cmp $op1, $op2" %}
14527
14528 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14529
14530 ins_pipe(icmp_reg_imm);
14531 %}
14532
14533 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14534 %{
14535 match(Set cr (CmpL op1 op2));
14536
14537 effect(DEF cr, USE op1);
14538
14539 ins_cost(INSN_COST * 2);
14540 format %{ "cmp $op1, $op2" %}
14541
14542 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14543
14544 ins_pipe(icmp_reg_imm);
14545 %}
14546
14547 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14548 %{
14549 match(Set cr (CmpUL op1 op2));
14550
14551 effect(DEF cr, USE op1, USE op2);
14552
14553 ins_cost(INSN_COST);
14554 format %{ "cmp $op1, $op2" %}
14555
14556 ins_encode(aarch64_enc_cmp(op1, op2));
14557
14558 ins_pipe(icmp_reg_reg);
14559 %}
14560
14561 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14562 %{
14563 match(Set cr (CmpUL op1 zero));
14564
14565 effect(DEF cr, USE op1);
14566
14567 ins_cost(INSN_COST);
14568 format %{ "tst $op1" %}
14569
14570 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14571
14572 ins_pipe(icmp_reg_imm);
14573 %}
14574
14575 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14576 %{
14577 match(Set cr (CmpUL op1 op2));
14578
14579 effect(DEF cr, USE op1);
14580
14581 ins_cost(INSN_COST);
14582 format %{ "cmp $op1, $op2" %}
14583
14584 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14585
14586 ins_pipe(icmp_reg_imm);
14587 %}
14588
14589 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14590 %{
14591 match(Set cr (CmpUL op1 op2));
14592
14593 effect(DEF cr, USE op1);
14594
14595 ins_cost(INSN_COST * 2);
14596 format %{ "cmp $op1, $op2" %}
14597
14598 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14599
14600 ins_pipe(icmp_reg_imm);
14601 %}
14602
14603 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14604 %{
14605 match(Set cr (CmpP op1 op2));
14606
14607 effect(DEF cr, USE op1, USE op2);
14608
14609 ins_cost(INSN_COST);
14610 format %{ "cmp $op1, $op2\t // ptr" %}
14611
14612 ins_encode(aarch64_enc_cmpp(op1, op2));
14613
14614 ins_pipe(icmp_reg_reg);
14615 %}
14616
14617 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14618 %{
14619 match(Set cr (CmpN op1 op2));
14620
14621 effect(DEF cr, USE op1, USE op2);
14622
14623 ins_cost(INSN_COST);
14624 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14625
14626 ins_encode(aarch64_enc_cmpn(op1, op2));
14627
14628 ins_pipe(icmp_reg_reg);
14629 %}
14630
14631 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14632 %{
14633 match(Set cr (CmpP op1 zero));
14634
14635 effect(DEF cr, USE op1, USE zero);
14636
14637 ins_cost(INSN_COST);
14638 format %{ "cmp $op1, 0\t // ptr" %}
14639
14640 ins_encode(aarch64_enc_testp(op1));
14641
14642 ins_pipe(icmp_reg_imm);
14643 %}
14644
14645 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14646 %{
14647 match(Set cr (CmpN op1 zero));
14648
14649 effect(DEF cr, USE op1, USE zero);
14650
14651 ins_cost(INSN_COST);
14652 format %{ "cmp $op1, 0\t // compressed ptr" %}
14653
14654 ins_encode(aarch64_enc_testn(op1));
14655
14656 ins_pipe(icmp_reg_imm);
14657 %}
14658
14659 // FP comparisons
14660 //
14661 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14662 // using normal cmpOp. See declaration of rFlagsReg for details.
14663
14664 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14665 %{
14666 match(Set cr (CmpF src1 src2));
14667
14668 ins_cost(3 * INSN_COST);
14669 format %{ "fcmps $src1, $src2" %}
14670
14671 ins_encode %{
14672 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14673 %}
14674
14675 ins_pipe(pipe_class_compare);
14676 %}
14677
14678 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14679 %{
14680 match(Set cr (CmpF src1 src2));
14681
14682 ins_cost(3 * INSN_COST);
14683 format %{ "fcmps $src1, 0.0" %}
14684
14685 ins_encode %{
14686 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14687 %}
14688
14689 ins_pipe(pipe_class_compare);
14690 %}
14691 // FROM HERE
14692
14693 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14694 %{
14695 match(Set cr (CmpD src1 src2));
14696
14697 ins_cost(3 * INSN_COST);
14698 format %{ "fcmpd $src1, $src2" %}
14699
14700 ins_encode %{
14701 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14702 %}
14703
14704 ins_pipe(pipe_class_compare);
14705 %}
14706
14707 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14708 %{
14709 match(Set cr (CmpD src1 src2));
14710
14711 ins_cost(3 * INSN_COST);
14712 format %{ "fcmpd $src1, 0.0" %}
14713
14714 ins_encode %{
14715 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14716 %}
14717
14718 ins_pipe(pipe_class_compare);
14719 %}
14720
14721 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14722 %{
14723 match(Set dst (CmpF3 src1 src2));
14724 effect(KILL cr);
14725
14726 ins_cost(5 * INSN_COST);
14727 format %{ "fcmps $src1, $src2\n\t"
14728 "csinvw($dst, zr, zr, eq\n\t"
14729 "csnegw($dst, $dst, $dst, lt)"
14730 %}
14731
14732 ins_encode %{
14733 Label done;
14734 FloatRegister s1 = as_FloatRegister($src1$$reg);
14735 FloatRegister s2 = as_FloatRegister($src2$$reg);
14736 Register d = as_Register($dst$$reg);
14737 __ fcmps(s1, s2);
14738 // installs 0 if EQ else -1
14739 __ csinvw(d, zr, zr, Assembler::EQ);
14740 // keeps -1 if less or unordered else installs 1
14741 __ csnegw(d, d, d, Assembler::LT);
14742 __ bind(done);
14743 %}
14744
14745 ins_pipe(pipe_class_default);
14746
14747 %}
14748
14749 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14750 %{
14751 match(Set dst (CmpD3 src1 src2));
14752 effect(KILL cr);
14753
14754 ins_cost(5 * INSN_COST);
14755 format %{ "fcmpd $src1, $src2\n\t"
14756 "csinvw($dst, zr, zr, eq\n\t"
14757 "csnegw($dst, $dst, $dst, lt)"
14758 %}
14759
14760 ins_encode %{
14761 Label done;
14762 FloatRegister s1 = as_FloatRegister($src1$$reg);
14763 FloatRegister s2 = as_FloatRegister($src2$$reg);
14764 Register d = as_Register($dst$$reg);
14765 __ fcmpd(s1, s2);
14766 // installs 0 if EQ else -1
14767 __ csinvw(d, zr, zr, Assembler::EQ);
14768 // keeps -1 if less or unordered else installs 1
14769 __ csnegw(d, d, d, Assembler::LT);
14770 __ bind(done);
14771 %}
14772 ins_pipe(pipe_class_default);
14773
14774 %}
14775
14776 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14777 %{
14778 match(Set dst (CmpF3 src1 zero));
14779 effect(KILL cr);
14780
14781 ins_cost(5 * INSN_COST);
14782 format %{ "fcmps $src1, 0.0\n\t"
14783 "csinvw($dst, zr, zr, eq\n\t"
14784 "csnegw($dst, $dst, $dst, lt)"
14785 %}
14786
14787 ins_encode %{
14788 Label done;
14789 FloatRegister s1 = as_FloatRegister($src1$$reg);
14790 Register d = as_Register($dst$$reg);
14791 __ fcmps(s1, 0.0);
14792 // installs 0 if EQ else -1
14793 __ csinvw(d, zr, zr, Assembler::EQ);
14794 // keeps -1 if less or unordered else installs 1
14795 __ csnegw(d, d, d, Assembler::LT);
14796 __ bind(done);
14797 %}
14798
14799 ins_pipe(pipe_class_default);
14800
14801 %}
14802
14803 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14804 %{
14805 match(Set dst (CmpD3 src1 zero));
14806 effect(KILL cr);
14807
14808 ins_cost(5 * INSN_COST);
14809 format %{ "fcmpd $src1, 0.0\n\t"
14810 "csinvw($dst, zr, zr, eq\n\t"
14811 "csnegw($dst, $dst, $dst, lt)"
14812 %}
14813
14814 ins_encode %{
14815 Label done;
14816 FloatRegister s1 = as_FloatRegister($src1$$reg);
14817 Register d = as_Register($dst$$reg);
14818 __ fcmpd(s1, 0.0);
14819 // installs 0 if EQ else -1
14820 __ csinvw(d, zr, zr, Assembler::EQ);
14821 // keeps -1 if less or unordered else installs 1
14822 __ csnegw(d, d, d, Assembler::LT);
14823 __ bind(done);
14824 %}
14825 ins_pipe(pipe_class_default);
14826
14827 %}
14828
14829 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14830 %{
14831 match(Set dst (CmpLTMask p q));
14832 effect(KILL cr);
14833
14834 ins_cost(3 * INSN_COST);
14835
14836 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14837 "csetw $dst, lt\n\t"
14838 "subw $dst, zr, $dst"
14839 %}
14840
14841 ins_encode %{
14842 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14843 __ csetw(as_Register($dst$$reg), Assembler::LT);
14844 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14845 %}
14846
14847 ins_pipe(ialu_reg_reg);
14848 %}
14849
14850 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14851 %{
14852 match(Set dst (CmpLTMask src zero));
14853 effect(KILL cr);
14854
14855 ins_cost(INSN_COST);
14856
14857 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14858
14859 ins_encode %{
14860 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14861 %}
14862
14863 ins_pipe(ialu_reg_shift);
14864 %}
14865
14866 // ============================================================================
14867 // Max and Min
14868
14869 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14870
14871 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14872 %{
14873 effect(DEF cr, USE src);
14874 ins_cost(INSN_COST);
14875 format %{ "cmpw $src, 0" %}
14876
14877 ins_encode %{
14878 __ cmpw($src$$Register, 0);
14879 %}
14880 ins_pipe(icmp_reg_imm);
14881 %}
14882
14883 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14884 %{
14885 match(Set dst (MinI src1 src2));
14886 ins_cost(INSN_COST * 3);
14887
14888 expand %{
14889 rFlagsReg cr;
14890 compI_reg_reg(cr, src1, src2);
14891 cmovI_reg_reg_lt(dst, src1, src2, cr);
14892 %}
14893 %}
14894
14895 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14896 %{
14897 match(Set dst (MaxI src1 src2));
14898 ins_cost(INSN_COST * 3);
14899
14900 expand %{
14901 rFlagsReg cr;
14902 compI_reg_reg(cr, src1, src2);
14903 cmovI_reg_reg_gt(dst, src1, src2, cr);
14904 %}
14905 %}
14906
14907
14908 // ============================================================================
14909 // Branch Instructions
14910
14911 // Direct Branch.
14912 instruct branch(label lbl)
14913 %{
14914 match(Goto);
14915
14916 effect(USE lbl);
14917
14918 ins_cost(BRANCH_COST);
14919 format %{ "b $lbl" %}
14920
14921 ins_encode(aarch64_enc_b(lbl));
14922
14923 ins_pipe(pipe_branch);
14924 %}
14925
14926 // Conditional Near Branch
14927 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14928 %{
14929 // Same match rule as `branchConFar'.
14930 match(If cmp cr);
14931
14932 effect(USE lbl);
14933
14934 ins_cost(BRANCH_COST);
14935 // If set to 1 this indicates that the current instruction is a
14936 // short variant of a long branch. This avoids using this
14937 // instruction in first-pass matching. It will then only be used in
14938 // the `Shorten_branches' pass.
14939 // ins_short_branch(1);
14940 format %{ "b$cmp $lbl" %}
14941
14942 ins_encode(aarch64_enc_br_con(cmp, lbl));
14943
14944 ins_pipe(pipe_branch_cond);
14945 %}
14946
14947 // Conditional Near Branch Unsigned
14948 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14949 %{
14950 // Same match rule as `branchConFar'.
14951 match(If cmp cr);
14952
14953 effect(USE lbl);
14954
14955 ins_cost(BRANCH_COST);
14956 // If set to 1 this indicates that the current instruction is a
14957 // short variant of a long branch. This avoids using this
14958 // instruction in first-pass matching. It will then only be used in
14959 // the `Shorten_branches' pass.
14960 // ins_short_branch(1);
14961 format %{ "b$cmp $lbl\t# unsigned" %}
14962
14963 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14964
14965 ins_pipe(pipe_branch_cond);
14966 %}
14967
14968 // Make use of CBZ and CBNZ. These instructions, as well as being
14969 // shorter than (cmp; branch), have the additional benefit of not
14970 // killing the flags.
14971
14972 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14973 match(If cmp (CmpI op1 op2));
14974 effect(USE labl);
14975
14976 ins_cost(BRANCH_COST);
14977 format %{ "cbw$cmp $op1, $labl" %}
14978 ins_encode %{
14979 Label* L = $labl$$label;
14980 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14981 if (cond == Assembler::EQ)
14982 __ cbzw($op1$$Register, *L);
14983 else
14984 __ cbnzw($op1$$Register, *L);
14985 %}
14986 ins_pipe(pipe_cmp_branch);
14987 %}
14988
14989 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14990 match(If cmp (CmpL op1 op2));
14991 effect(USE labl);
14992
14993 ins_cost(BRANCH_COST);
14994 format %{ "cb$cmp $op1, $labl" %}
14995 ins_encode %{
14996 Label* L = $labl$$label;
14997 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14998 if (cond == Assembler::EQ)
14999 __ cbz($op1$$Register, *L);
15000 else
15001 __ cbnz($op1$$Register, *L);
15002 %}
15003 ins_pipe(pipe_cmp_branch);
15004 %}
15005
15006 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15007 match(If cmp (CmpP op1 op2));
15008 effect(USE labl);
15009
15010 ins_cost(BRANCH_COST);
15011 format %{ "cb$cmp $op1, $labl" %}
15012 ins_encode %{
15013 Label* L = $labl$$label;
15014 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15015 if (cond == Assembler::EQ)
15016 __ cbz($op1$$Register, *L);
15017 else
15018 __ cbnz($op1$$Register, *L);
15019 %}
15020 ins_pipe(pipe_cmp_branch);
15021 %}
15022
15023 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15024 match(If cmp (CmpN op1 op2));
15025 effect(USE labl);
15026
15027 ins_cost(BRANCH_COST);
15028 format %{ "cbw$cmp $op1, $labl" %}
15029 ins_encode %{
15030 Label* L = $labl$$label;
15031 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15032 if (cond == Assembler::EQ)
15033 __ cbzw($op1$$Register, *L);
15034 else
15035 __ cbnzw($op1$$Register, *L);
15036 %}
15037 ins_pipe(pipe_cmp_branch);
15038 %}
15039
15040 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15041 match(If cmp (CmpP (DecodeN oop) zero));
15042 effect(USE labl);
15043
15044 ins_cost(BRANCH_COST);
15045 format %{ "cb$cmp $oop, $labl" %}
15046 ins_encode %{
15047 Label* L = $labl$$label;
15048 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15049 if (cond == Assembler::EQ)
15050 __ cbzw($oop$$Register, *L);
15051 else
15052 __ cbnzw($oop$$Register, *L);
15053 %}
15054 ins_pipe(pipe_cmp_branch);
15055 %}
15056
15057 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15058 match(If cmp (CmpU op1 op2));
15059 effect(USE labl);
15060
15061 ins_cost(BRANCH_COST);
15062 format %{ "cbw$cmp $op1, $labl" %}
15063 ins_encode %{
15064 Label* L = $labl$$label;
15065 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15066 if (cond == Assembler::EQ || cond == Assembler::LS) {
15067 __ cbzw($op1$$Register, *L);
15068 } else {
15069 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15070 __ cbnzw($op1$$Register, *L);
15071 }
15072 %}
15073 ins_pipe(pipe_cmp_branch);
15074 %}
15075
15076 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15077 match(If cmp (CmpUL op1 op2));
15078 effect(USE labl);
15079
15080 ins_cost(BRANCH_COST);
15081 format %{ "cb$cmp $op1, $labl" %}
15082 ins_encode %{
15083 Label* L = $labl$$label;
15084 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15085 if (cond == Assembler::EQ || cond == Assembler::LS) {
15086 __ cbz($op1$$Register, *L);
15087 } else {
15088 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15089 __ cbnz($op1$$Register, *L);
15090 }
15091 %}
15092 ins_pipe(pipe_cmp_branch);
15093 %}
15094
15095 // Test bit and Branch
15096
15097 // Patterns for short (< 32KiB) variants
15098 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15099 match(If cmp (CmpL op1 op2));
15100 effect(USE labl);
15101
15102 ins_cost(BRANCH_COST);
15103 format %{ "cb$cmp $op1, $labl # long" %}
15104 ins_encode %{
15105 Label* L = $labl$$label;
15106 Assembler::Condition cond =
15107 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15108 __ tbr(cond, $op1$$Register, 63, *L);
15109 %}
15110 ins_pipe(pipe_cmp_branch);
15111 ins_short_branch(1);
15112 %}
15113
15114 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15115 match(If cmp (CmpI op1 op2));
15116 effect(USE labl);
15117
15118 ins_cost(BRANCH_COST);
15119 format %{ "cb$cmp $op1, $labl # int" %}
15120 ins_encode %{
15121 Label* L = $labl$$label;
15122 Assembler::Condition cond =
15123 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15124 __ tbr(cond, $op1$$Register, 31, *L);
15125 %}
15126 ins_pipe(pipe_cmp_branch);
15127 ins_short_branch(1);
15128 %}
15129
15130 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15131 match(If cmp (CmpL (AndL op1 op2) op3));
15132 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15133 effect(USE labl);
15134
15135 ins_cost(BRANCH_COST);
15136 format %{ "tb$cmp $op1, $op2, $labl" %}
15137 ins_encode %{
15138 Label* L = $labl$$label;
15139 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15140 int bit = exact_log2_long($op2$$constant);
15141 __ tbr(cond, $op1$$Register, bit, *L);
15142 %}
15143 ins_pipe(pipe_cmp_branch);
15144 ins_short_branch(1);
15145 %}
15146
15147 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15148 match(If cmp (CmpI (AndI op1 op2) op3));
15149 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15150 effect(USE labl);
15151
15152 ins_cost(BRANCH_COST);
15153 format %{ "tb$cmp $op1, $op2, $labl" %}
15154 ins_encode %{
15155 Label* L = $labl$$label;
15156 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15157 int bit = exact_log2((juint)$op2$$constant);
15158 __ tbr(cond, $op1$$Register, bit, *L);
15159 %}
15160 ins_pipe(pipe_cmp_branch);
15161 ins_short_branch(1);
15162 %}
15163
15164 // And far variants
15165 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15166 match(If cmp (CmpL op1 op2));
15167 effect(USE labl);
15168
15169 ins_cost(BRANCH_COST);
15170 format %{ "cb$cmp $op1, $labl # long" %}
15171 ins_encode %{
15172 Label* L = $labl$$label;
15173 Assembler::Condition cond =
15174 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15175 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15176 %}
15177 ins_pipe(pipe_cmp_branch);
15178 %}
15179
15180 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15181 match(If cmp (CmpI op1 op2));
15182 effect(USE labl);
15183
15184 ins_cost(BRANCH_COST);
15185 format %{ "cb$cmp $op1, $labl # int" %}
15186 ins_encode %{
15187 Label* L = $labl$$label;
15188 Assembler::Condition cond =
15189 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15190 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15191 %}
15192 ins_pipe(pipe_cmp_branch);
15193 %}
15194
15195 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15196 match(If cmp (CmpL (AndL op1 op2) op3));
15197 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15198 effect(USE labl);
15199
15200 ins_cost(BRANCH_COST);
15201 format %{ "tb$cmp $op1, $op2, $labl" %}
15202 ins_encode %{
15203 Label* L = $labl$$label;
15204 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15205 int bit = exact_log2_long($op2$$constant);
15206 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15207 %}
15208 ins_pipe(pipe_cmp_branch);
15209 %}
15210
15211 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15212 match(If cmp (CmpI (AndI op1 op2) op3));
15213 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15214 effect(USE labl);
15215
15216 ins_cost(BRANCH_COST);
15217 format %{ "tb$cmp $op1, $op2, $labl" %}
15218 ins_encode %{
15219 Label* L = $labl$$label;
15220 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15221 int bit = exact_log2((juint)$op2$$constant);
15222 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15223 %}
15224 ins_pipe(pipe_cmp_branch);
15225 %}
15226
15227 // Test bits
15228
15229 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15230 match(Set cr (CmpL (AndL op1 op2) op3));
15231 predicate(Assembler::operand_valid_for_logical_immediate
15232 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15233
15234 ins_cost(INSN_COST);
15235 format %{ "tst $op1, $op2 # long" %}
15236 ins_encode %{
15237 __ tst($op1$$Register, $op2$$constant);
15238 %}
15239 ins_pipe(ialu_reg_reg);
15240 %}
15241
15242 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15243 match(Set cr (CmpI (AndI op1 op2) op3));
15244 predicate(Assembler::operand_valid_for_logical_immediate
15245 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15246
15247 ins_cost(INSN_COST);
15248 format %{ "tst $op1, $op2 # int" %}
15249 ins_encode %{
15250 __ tstw($op1$$Register, $op2$$constant);
15251 %}
15252 ins_pipe(ialu_reg_reg);
15253 %}
15254
15255 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15256 match(Set cr (CmpL (AndL op1 op2) op3));
15257
15258 ins_cost(INSN_COST);
15259 format %{ "tst $op1, $op2 # long" %}
15260 ins_encode %{
15261 __ tst($op1$$Register, $op2$$Register);
15262 %}
15263 ins_pipe(ialu_reg_reg);
15264 %}
15265
15266 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15267 match(Set cr (CmpI (AndI op1 op2) op3));
15268
15269 ins_cost(INSN_COST);
15270 format %{ "tstw $op1, $op2 # int" %}
15271 ins_encode %{
15272 __ tstw($op1$$Register, $op2$$Register);
15273 %}
15274 ins_pipe(ialu_reg_reg);
15275 %}
15276
15277
15278 // Conditional Far Branch
15279 // Conditional Far Branch Unsigned
15280 // TODO: fixme
15281
15282 // counted loop end branch near
15283 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15284 %{
15285 match(CountedLoopEnd cmp cr);
15286
15287 effect(USE lbl);
15288
15289 ins_cost(BRANCH_COST);
15290 // short variant.
15291 // ins_short_branch(1);
15292 format %{ "b$cmp $lbl \t// counted loop end" %}
15293
15294 ins_encode(aarch64_enc_br_con(cmp, lbl));
15295
15296 ins_pipe(pipe_branch);
15297 %}
15298
15299 // counted loop end branch far
15300 // TODO: fixme
15301
15302 // ============================================================================
15303 // inlined locking and unlocking
15304
15305 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15306 %{
15307 match(Set cr (FastLock object box));
15308 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15309
15310 ins_cost(5 * INSN_COST);
15311 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15312
15313 ins_encode %{
15314 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15315 %}
15316
15317 ins_pipe(pipe_serial);
15318 %}
15319
15320 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15321 %{
15322 match(Set cr (FastUnlock object box));
15323 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15324
15325 ins_cost(5 * INSN_COST);
15326 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15327
15328 ins_encode %{
15329 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15330 %}
15331
15332 ins_pipe(pipe_serial);
15333 %}
15334
15335 // ============================================================================
15336 // Safepoint Instructions
15337
15338 // TODO
15339 // provide a near and far version of this code
15340
15341 instruct safePoint(rFlagsReg cr, iRegP poll)
15342 %{
15343 match(SafePoint poll);
15344 effect(KILL cr);
15345
15346 format %{
15347 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15348 %}
15349 ins_encode %{
15350 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15351 %}
15352 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15353 %}
15354
15355
15356 // ============================================================================
15357 // Procedure Call/Return Instructions
15358
15359 // Call Java Static Instruction
15360
15361 instruct CallStaticJavaDirect(method meth)
15362 %{
15363 match(CallStaticJava);
15364
15365 effect(USE meth);
15366
15367 ins_cost(CALL_COST);
15368
15369 format %{ "call,static $meth \t// ==> " %}
15370
15371 ins_encode(aarch64_enc_java_static_call(meth),
15372 aarch64_enc_call_epilog);
15373
15374 ins_pipe(pipe_class_call);
15375 %}
15376
15377 // TO HERE
15378
15379 // Call Java Dynamic Instruction
15380 instruct CallDynamicJavaDirect(method meth)
15381 %{
15382 match(CallDynamicJava);
15383
15384 effect(USE meth);
15385
15386 ins_cost(CALL_COST);
15387
15388 format %{ "CALL,dynamic $meth \t// ==> " %}
15389
15390 ins_encode(aarch64_enc_java_dynamic_call(meth),
15391 aarch64_enc_call_epilog);
15392
15393 ins_pipe(pipe_class_call);
15394 %}
15395
15396 // Call Runtime Instruction
15397
15398 instruct CallRuntimeDirect(method meth)
15399 %{
15400 match(CallRuntime);
15401
15402 effect(USE meth);
15403
15404 ins_cost(CALL_COST);
15405
15406 format %{ "CALL, runtime $meth" %}
15407
15408 ins_encode( aarch64_enc_java_to_runtime(meth) );
15409
15410 ins_pipe(pipe_class_call);
15411 %}
15412
15413 // Call Runtime Instruction
15414
15415 instruct CallLeafDirect(method meth)
15416 %{
15417 match(CallLeaf);
15418
15419 effect(USE meth);
15420
15421 ins_cost(CALL_COST);
15422
15423 format %{ "CALL, runtime leaf $meth" %}
15424
15425 ins_encode( aarch64_enc_java_to_runtime(meth) );
15426
15427 ins_pipe(pipe_class_call);
15428 %}
15429
15430 // Call Runtime Instruction without safepoint and with vector arguments
15431 instruct CallLeafDirectVector(method meth)
15432 %{
15433 match(CallLeafVector);
15434
15435 effect(USE meth);
15436
15437 ins_cost(CALL_COST);
15438
15439 format %{ "CALL, runtime leaf vector $meth" %}
15440
15441 ins_encode(aarch64_enc_java_to_runtime(meth));
15442
15443 ins_pipe(pipe_class_call);
15444 %}
15445
15446 // Call Runtime Instruction
15447
15448 instruct CallLeafNoFPDirect(method meth)
15449 %{
15450 match(CallLeafNoFP);
15451
15452 effect(USE meth);
15453
15454 ins_cost(CALL_COST);
15455
15456 format %{ "CALL, runtime leaf nofp $meth" %}
15457
15458 ins_encode( aarch64_enc_java_to_runtime(meth) );
15459
15460 ins_pipe(pipe_class_call);
15461 %}
15462
15463 // Tail Call; Jump from runtime stub to Java code.
15464 // Also known as an 'interprocedural jump'.
15465 // Target of jump will eventually return to caller.
15466 // TailJump below removes the return address.
15467 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15468 // emitted just above the TailCall which has reset rfp to the caller state.
15469 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15470 %{
15471 match(TailCall jump_target method_ptr);
15472
15473 ins_cost(CALL_COST);
15474
15475 format %{ "br $jump_target\t# $method_ptr holds method" %}
15476
15477 ins_encode(aarch64_enc_tail_call(jump_target));
15478
15479 ins_pipe(pipe_class_call);
15480 %}
15481
15482 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15483 %{
15484 match(TailJump jump_target ex_oop);
15485
15486 ins_cost(CALL_COST);
15487
15488 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15489
15490 ins_encode(aarch64_enc_tail_jmp(jump_target));
15491
15492 ins_pipe(pipe_class_call);
15493 %}
15494
15495 // Forward exception.
15496 instruct ForwardExceptionjmp()
15497 %{
15498 match(ForwardException);
15499 ins_cost(CALL_COST);
15500
15501 format %{ "b forward_exception_stub" %}
15502 ins_encode %{
15503 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15504 %}
15505 ins_pipe(pipe_class_call);
15506 %}
15507
15508 // Create exception oop: created by stack-crawling runtime code.
15509 // Created exception is now available to this handler, and is setup
15510 // just prior to jumping to this handler. No code emitted.
15511 // TODO check
15512 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15513 instruct CreateException(iRegP_R0 ex_oop)
15514 %{
15515 match(Set ex_oop (CreateEx));
15516
15517 format %{ " -- \t// exception oop; no code emitted" %}
15518
15519 size(0);
15520
15521 ins_encode( /*empty*/ );
15522
15523 ins_pipe(pipe_class_empty);
15524 %}
15525
15526 // Rethrow exception: The exception oop will come in the first
15527 // argument position. Then JUMP (not call) to the rethrow stub code.
15528 instruct RethrowException() %{
15529 match(Rethrow);
15530 ins_cost(CALL_COST);
15531
15532 format %{ "b rethrow_stub" %}
15533
15534 ins_encode( aarch64_enc_rethrow() );
15535
15536 ins_pipe(pipe_class_call);
15537 %}
15538
15539
15540 // Return Instruction
15541 // epilog node loads ret address into lr as part of frame pop
15542 instruct Ret()
15543 %{
15544 match(Return);
15545
15546 format %{ "ret\t// return register" %}
15547
15548 ins_encode( aarch64_enc_ret() );
15549
15550 ins_pipe(pipe_branch);
15551 %}
15552
15553 // Die now.
15554 instruct ShouldNotReachHere() %{
15555 match(Halt);
15556
15557 ins_cost(CALL_COST);
15558 format %{ "ShouldNotReachHere" %}
15559
15560 ins_encode %{
15561 if (is_reachable()) {
15562 const char* str = __ code_string(_halt_reason);
15563 __ stop(str);
15564 }
15565 %}
15566
15567 ins_pipe(pipe_class_default);
15568 %}
15569
15570 // ============================================================================
15571 // Partial Subtype Check
15572 //
15573 // superklass array for an instance of the superklass. Set a hidden
15574 // internal cache on a hit (cache is checked with exposed code in
15575 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15576 // encoding ALSO sets flags.
15577
15578 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15579 %{
15580 match(Set result (PartialSubtypeCheck sub super));
15581 predicate(!UseSecondarySupersTable);
15582 effect(KILL cr, KILL temp);
15583
15584 ins_cost(20 * INSN_COST); // slightly larger than the next version
15585 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15586
15587 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15588
15589 opcode(0x1); // Force zero of result reg on hit
15590
15591 ins_pipe(pipe_class_memory);
15592 %}
15593
15594 // Two versions of partialSubtypeCheck, both used when we need to
15595 // search for a super class in the secondary supers array. The first
15596 // is used when we don't know _a priori_ the class being searched
15597 // for. The second, far more common, is used when we do know: this is
15598 // used for instanceof, checkcast, and any case where C2 can determine
15599 // it by constant propagation.
15600
15601 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15602 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15603 rFlagsReg cr)
15604 %{
15605 match(Set result (PartialSubtypeCheck sub super));
15606 predicate(UseSecondarySupersTable);
15607 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15608
15609 ins_cost(10 * INSN_COST); // slightly larger than the next version
15610 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15611
15612 ins_encode %{
15613 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15614 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15615 $vtemp$$FloatRegister,
15616 $result$$Register, /*L_success*/nullptr);
15617 %}
15618
15619 ins_pipe(pipe_class_memory);
15620 %}
15621
15622 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15623 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15624 rFlagsReg cr)
15625 %{
15626 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15627 predicate(UseSecondarySupersTable);
15628 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15629
15630 ins_cost(5 * INSN_COST); // smaller than the next version
15631 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15632
15633 ins_encode %{
15634 bool success = false;
15635 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15636 if (InlineSecondarySupersTest) {
15637 success =
15638 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15639 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15640 $vtemp$$FloatRegister,
15641 $result$$Register,
15642 super_klass_slot);
15643 } else {
15644 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15645 success = (call != nullptr);
15646 }
15647 if (!success) {
15648 ciEnv::current()->record_failure("CodeCache is full");
15649 return;
15650 }
15651 %}
15652
15653 ins_pipe(pipe_class_memory);
15654 %}
15655
15656 // Intrisics for String.compareTo()
15657
15658 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15659 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15660 %{
15661 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15662 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15663 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15664
15665 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15666 ins_encode %{
15667 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15668 __ string_compare($str1$$Register, $str2$$Register,
15669 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15670 $tmp1$$Register, $tmp2$$Register,
15671 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15672 %}
15673 ins_pipe(pipe_class_memory);
15674 %}
15675
15676 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15677 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15678 %{
15679 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15680 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15681 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15682
15683 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15684 ins_encode %{
15685 __ string_compare($str1$$Register, $str2$$Register,
15686 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15687 $tmp1$$Register, $tmp2$$Register,
15688 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15689 %}
15690 ins_pipe(pipe_class_memory);
15691 %}
15692
15693 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15694 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15695 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15696 %{
15697 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15698 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15699 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15700 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15701
15702 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15703 ins_encode %{
15704 __ string_compare($str1$$Register, $str2$$Register,
15705 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15706 $tmp1$$Register, $tmp2$$Register,
15707 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15708 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15709 %}
15710 ins_pipe(pipe_class_memory);
15711 %}
15712
15713 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15714 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15715 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15716 %{
15717 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15718 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15719 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15720 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15721
15722 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15723 ins_encode %{
15724 __ string_compare($str1$$Register, $str2$$Register,
15725 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15726 $tmp1$$Register, $tmp2$$Register,
15727 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15728 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15729 %}
15730 ins_pipe(pipe_class_memory);
15731 %}
15732
15733 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15734 // these string_compare variants as NEON register type for convenience so that the prototype of
15735 // string_compare can be shared with all variants.
15736
15737 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15738 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15739 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15740 pRegGov_P1 pgtmp2, rFlagsReg cr)
15741 %{
15742 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15743 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15744 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15745 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15746
15747 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15748 ins_encode %{
15749 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15750 __ string_compare($str1$$Register, $str2$$Register,
15751 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15752 $tmp1$$Register, $tmp2$$Register,
15753 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15754 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15755 StrIntrinsicNode::LL);
15756 %}
15757 ins_pipe(pipe_class_memory);
15758 %}
15759
15760 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15761 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15762 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15763 pRegGov_P1 pgtmp2, rFlagsReg cr)
15764 %{
15765 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15766 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15767 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15768 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15769
15770 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15771 ins_encode %{
15772 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15773 __ string_compare($str1$$Register, $str2$$Register,
15774 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15775 $tmp1$$Register, $tmp2$$Register,
15776 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15777 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15778 StrIntrinsicNode::LU);
15779 %}
15780 ins_pipe(pipe_class_memory);
15781 %}
15782
15783 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15784 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15785 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15786 pRegGov_P1 pgtmp2, rFlagsReg cr)
15787 %{
15788 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15789 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15790 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15791 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15792
15793 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15794 ins_encode %{
15795 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15796 __ string_compare($str1$$Register, $str2$$Register,
15797 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15798 $tmp1$$Register, $tmp2$$Register,
15799 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15800 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15801 StrIntrinsicNode::UL);
15802 %}
15803 ins_pipe(pipe_class_memory);
15804 %}
15805
15806 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15807 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15808 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15809 pRegGov_P1 pgtmp2, rFlagsReg cr)
15810 %{
15811 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15812 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15813 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15814 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15815
15816 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15817 ins_encode %{
15818 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15819 __ string_compare($str1$$Register, $str2$$Register,
15820 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15821 $tmp1$$Register, $tmp2$$Register,
15822 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15823 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15824 StrIntrinsicNode::UU);
15825 %}
15826 ins_pipe(pipe_class_memory);
15827 %}
15828
15829 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15830 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15831 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15832 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15833 %{
15834 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15835 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15836 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15837 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15838 TEMP vtmp0, TEMP vtmp1, KILL cr);
15839 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15840 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15841
15842 ins_encode %{
15843 __ string_indexof($str1$$Register, $str2$$Register,
15844 $cnt1$$Register, $cnt2$$Register,
15845 $tmp1$$Register, $tmp2$$Register,
15846 $tmp3$$Register, $tmp4$$Register,
15847 $tmp5$$Register, $tmp6$$Register,
15848 -1, $result$$Register, StrIntrinsicNode::UU);
15849 %}
15850 ins_pipe(pipe_class_memory);
15851 %}
15852
15853 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15854 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15855 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15856 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15857 %{
15858 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15859 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15860 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15861 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15862 TEMP vtmp0, TEMP vtmp1, KILL cr);
15863 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15864 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15865
15866 ins_encode %{
15867 __ string_indexof($str1$$Register, $str2$$Register,
15868 $cnt1$$Register, $cnt2$$Register,
15869 $tmp1$$Register, $tmp2$$Register,
15870 $tmp3$$Register, $tmp4$$Register,
15871 $tmp5$$Register, $tmp6$$Register,
15872 -1, $result$$Register, StrIntrinsicNode::LL);
15873 %}
15874 ins_pipe(pipe_class_memory);
15875 %}
15876
15877 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15878 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15879 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15880 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15881 %{
15882 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15883 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15884 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15885 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15886 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15887 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15888 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15889
15890 ins_encode %{
15891 __ string_indexof($str1$$Register, $str2$$Register,
15892 $cnt1$$Register, $cnt2$$Register,
15893 $tmp1$$Register, $tmp2$$Register,
15894 $tmp3$$Register, $tmp4$$Register,
15895 $tmp5$$Register, $tmp6$$Register,
15896 -1, $result$$Register, StrIntrinsicNode::UL);
15897 %}
15898 ins_pipe(pipe_class_memory);
15899 %}
15900
15901 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15902 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15903 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15904 %{
15905 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15906 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15907 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15908 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15909 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15910 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15911
15912 ins_encode %{
15913 int icnt2 = (int)$int_cnt2$$constant;
15914 __ string_indexof($str1$$Register, $str2$$Register,
15915 $cnt1$$Register, zr,
15916 $tmp1$$Register, $tmp2$$Register,
15917 $tmp3$$Register, $tmp4$$Register, zr, zr,
15918 icnt2, $result$$Register, StrIntrinsicNode::UU);
15919 %}
15920 ins_pipe(pipe_class_memory);
15921 %}
15922
15923 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15924 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15925 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15926 %{
15927 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15928 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15929 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15930 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15931 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15932 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15933
15934 ins_encode %{
15935 int icnt2 = (int)$int_cnt2$$constant;
15936 __ string_indexof($str1$$Register, $str2$$Register,
15937 $cnt1$$Register, zr,
15938 $tmp1$$Register, $tmp2$$Register,
15939 $tmp3$$Register, $tmp4$$Register, zr, zr,
15940 icnt2, $result$$Register, StrIntrinsicNode::LL);
15941 %}
15942 ins_pipe(pipe_class_memory);
15943 %}
15944
15945 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15946 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15947 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15948 %{
15949 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15950 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15951 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15952 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15953 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15954 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15955
15956 ins_encode %{
15957 int icnt2 = (int)$int_cnt2$$constant;
15958 __ string_indexof($str1$$Register, $str2$$Register,
15959 $cnt1$$Register, zr,
15960 $tmp1$$Register, $tmp2$$Register,
15961 $tmp3$$Register, $tmp4$$Register, zr, zr,
15962 icnt2, $result$$Register, StrIntrinsicNode::UL);
15963 %}
15964 ins_pipe(pipe_class_memory);
15965 %}
15966
15967 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15968 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15969 iRegINoSp tmp3, rFlagsReg cr)
15970 %{
15971 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15972 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15973 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15974 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15975
15976 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15977
15978 ins_encode %{
15979 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15980 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15981 $tmp3$$Register);
15982 %}
15983 ins_pipe(pipe_class_memory);
15984 %}
15985
15986 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15987 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15988 iRegINoSp tmp3, rFlagsReg cr)
15989 %{
15990 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15991 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15992 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15993 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15994
15995 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15996
15997 ins_encode %{
15998 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15999 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16000 $tmp3$$Register);
16001 %}
16002 ins_pipe(pipe_class_memory);
16003 %}
16004
16005 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16006 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16007 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16008 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16009 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16010 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16011 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16012 ins_encode %{
16013 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16014 $result$$Register, $ztmp1$$FloatRegister,
16015 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16016 $ptmp$$PRegister, true /* isL */);
16017 %}
16018 ins_pipe(pipe_class_memory);
16019 %}
16020
16021 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16022 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16023 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16024 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16025 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16026 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16027 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16028 ins_encode %{
16029 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16030 $result$$Register, $ztmp1$$FloatRegister,
16031 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16032 $ptmp$$PRegister, false /* isL */);
16033 %}
16034 ins_pipe(pipe_class_memory);
16035 %}
16036
16037 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16038 iRegI_R0 result, rFlagsReg cr)
16039 %{
16040 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16041 match(Set result (StrEquals (Binary str1 str2) cnt));
16042 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16043
16044 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16045 ins_encode %{
16046 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16047 __ string_equals($str1$$Register, $str2$$Register,
16048 $result$$Register, $cnt$$Register);
16049 %}
16050 ins_pipe(pipe_class_memory);
16051 %}
16052
16053 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16054 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16055 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16056 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16057 iRegP_R10 tmp, rFlagsReg cr)
16058 %{
16059 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16060 match(Set result (AryEq ary1 ary2));
16061 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16062 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16063 TEMP vtmp6, TEMP vtmp7, KILL cr);
16064
16065 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16066 ins_encode %{
16067 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16068 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16069 $result$$Register, $tmp$$Register, 1);
16070 if (tpc == nullptr) {
16071 ciEnv::current()->record_failure("CodeCache is full");
16072 return;
16073 }
16074 %}
16075 ins_pipe(pipe_class_memory);
16076 %}
16077
16078 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16079 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16080 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16081 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16082 iRegP_R10 tmp, rFlagsReg cr)
16083 %{
16084 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16085 match(Set result (AryEq ary1 ary2));
16086 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16087 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16088 TEMP vtmp6, TEMP vtmp7, KILL cr);
16089
16090 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16091 ins_encode %{
16092 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16093 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16094 $result$$Register, $tmp$$Register, 2);
16095 if (tpc == nullptr) {
16096 ciEnv::current()->record_failure("CodeCache is full");
16097 return;
16098 }
16099 %}
16100 ins_pipe(pipe_class_memory);
16101 %}
16102
16103 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16104 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16105 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16106 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16107 %{
16108 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16109 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16110 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16111
16112 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16113 ins_encode %{
16114 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16115 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16116 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16117 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16118 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16119 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16120 (BasicType)$basic_type$$constant);
16121 if (tpc == nullptr) {
16122 ciEnv::current()->record_failure("CodeCache is full");
16123 return;
16124 }
16125 %}
16126 ins_pipe(pipe_class_memory);
16127 %}
16128
16129 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16130 %{
16131 match(Set result (CountPositives ary1 len));
16132 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16133 format %{ "count positives byte[] $ary1,$len -> $result" %}
16134 ins_encode %{
16135 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16136 if (tpc == nullptr) {
16137 ciEnv::current()->record_failure("CodeCache is full");
16138 return;
16139 }
16140 %}
16141 ins_pipe( pipe_slow );
16142 %}
16143
16144 // fast char[] to byte[] compression
16145 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16146 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16147 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16148 iRegI_R0 result, rFlagsReg cr)
16149 %{
16150 match(Set result (StrCompressedCopy src (Binary dst len)));
16151 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16152 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16153
16154 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16155 ins_encode %{
16156 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16157 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16158 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16159 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16160 %}
16161 ins_pipe(pipe_slow);
16162 %}
16163
16164 // fast byte[] to char[] inflation
16165 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16166 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16167 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16168 %{
16169 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16170 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16171 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16172 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16173
16174 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16175 ins_encode %{
16176 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16177 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16178 $vtmp2$$FloatRegister, $tmp$$Register);
16179 if (tpc == nullptr) {
16180 ciEnv::current()->record_failure("CodeCache is full");
16181 return;
16182 }
16183 %}
16184 ins_pipe(pipe_class_memory);
16185 %}
16186
16187 // encode char[] to byte[] in ISO_8859_1
16188 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16189 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16190 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16191 iRegI_R0 result, rFlagsReg cr)
16192 %{
16193 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16194 match(Set result (EncodeISOArray src (Binary dst len)));
16195 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16196 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16197
16198 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16199 ins_encode %{
16200 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16201 $result$$Register, false,
16202 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16203 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16204 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16205 %}
16206 ins_pipe(pipe_class_memory);
16207 %}
16208
16209 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16210 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16211 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16212 iRegI_R0 result, rFlagsReg cr)
16213 %{
16214 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16215 match(Set result (EncodeISOArray src (Binary dst len)));
16216 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16217 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16218
16219 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16220 ins_encode %{
16221 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16222 $result$$Register, true,
16223 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16224 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16225 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16226 %}
16227 ins_pipe(pipe_class_memory);
16228 %}
16229
16230 //----------------------------- CompressBits/ExpandBits ------------------------
16231
16232 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16233 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16234 match(Set dst (CompressBits src mask));
16235 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16236 format %{ "mov $tsrc, $src\n\t"
16237 "mov $tmask, $mask\n\t"
16238 "bext $tdst, $tsrc, $tmask\n\t"
16239 "mov $dst, $tdst"
16240 %}
16241 ins_encode %{
16242 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16243 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16244 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16245 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16246 %}
16247 ins_pipe(pipe_slow);
16248 %}
16249
16250 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16251 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16252 match(Set dst (CompressBits (LoadI mem) mask));
16253 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16254 format %{ "ldrs $tsrc, $mem\n\t"
16255 "ldrs $tmask, $mask\n\t"
16256 "bext $tdst, $tsrc, $tmask\n\t"
16257 "mov $dst, $tdst"
16258 %}
16259 ins_encode %{
16260 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16261 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16262 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16263 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16264 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16265 %}
16266 ins_pipe(pipe_slow);
16267 %}
16268
16269 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16270 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16271 match(Set dst (CompressBits src mask));
16272 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16273 format %{ "mov $tsrc, $src\n\t"
16274 "mov $tmask, $mask\n\t"
16275 "bext $tdst, $tsrc, $tmask\n\t"
16276 "mov $dst, $tdst"
16277 %}
16278 ins_encode %{
16279 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16280 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16281 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16282 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16283 %}
16284 ins_pipe(pipe_slow);
16285 %}
16286
16287 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16288 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16289 match(Set dst (CompressBits (LoadL mem) mask));
16290 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16291 format %{ "ldrd $tsrc, $mem\n\t"
16292 "ldrd $tmask, $mask\n\t"
16293 "bext $tdst, $tsrc, $tmask\n\t"
16294 "mov $dst, $tdst"
16295 %}
16296 ins_encode %{
16297 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16298 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16299 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16300 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16301 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16302 %}
16303 ins_pipe(pipe_slow);
16304 %}
16305
16306 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16307 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16308 match(Set dst (ExpandBits src mask));
16309 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16310 format %{ "mov $tsrc, $src\n\t"
16311 "mov $tmask, $mask\n\t"
16312 "bdep $tdst, $tsrc, $tmask\n\t"
16313 "mov $dst, $tdst"
16314 %}
16315 ins_encode %{
16316 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16317 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16318 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16319 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16320 %}
16321 ins_pipe(pipe_slow);
16322 %}
16323
16324 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16325 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16326 match(Set dst (ExpandBits (LoadI mem) mask));
16327 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16328 format %{ "ldrs $tsrc, $mem\n\t"
16329 "ldrs $tmask, $mask\n\t"
16330 "bdep $tdst, $tsrc, $tmask\n\t"
16331 "mov $dst, $tdst"
16332 %}
16333 ins_encode %{
16334 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16335 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16336 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16337 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16338 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16339 %}
16340 ins_pipe(pipe_slow);
16341 %}
16342
16343 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16344 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16345 match(Set dst (ExpandBits src mask));
16346 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16347 format %{ "mov $tsrc, $src\n\t"
16348 "mov $tmask, $mask\n\t"
16349 "bdep $tdst, $tsrc, $tmask\n\t"
16350 "mov $dst, $tdst"
16351 %}
16352 ins_encode %{
16353 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16354 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16355 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16356 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16357 %}
16358 ins_pipe(pipe_slow);
16359 %}
16360
16361
16362 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16363 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16364 match(Set dst (ExpandBits (LoadL mem) mask));
16365 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16366 format %{ "ldrd $tsrc, $mem\n\t"
16367 "ldrd $tmask, $mask\n\t"
16368 "bdep $tdst, $tsrc, $tmask\n\t"
16369 "mov $dst, $tdst"
16370 %}
16371 ins_encode %{
16372 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16373 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16374 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16375 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16376 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16377 %}
16378 ins_pipe(pipe_slow);
16379 %}
16380
16381 //----------------------------- Reinterpret ----------------------------------
16382 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16383 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16384 match(Set dst (ReinterpretHF2S src));
16385 format %{ "reinterpretHF2S $dst, $src" %}
16386 ins_encode %{
16387 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16388 %}
16389 ins_pipe(pipe_slow);
16390 %}
16391
16392 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16393 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16394 match(Set dst (ReinterpretS2HF src));
16395 format %{ "reinterpretS2HF $dst, $src" %}
16396 ins_encode %{
16397 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16398 %}
16399 ins_pipe(pipe_slow);
16400 %}
16401
16402 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16403 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16404 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16405 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16406 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16407 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16408 // can be omitted in this pattern, resulting in -
16409 // fcvt $dst, $src // Convert float to half-precision float
16410 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16411 %{
16412 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16413 format %{ "convF2HFAndS2HF $dst, $src" %}
16414 ins_encode %{
16415 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16416 %}
16417 ins_pipe(pipe_slow);
16418 %}
16419
16420 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16421 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16422 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16423 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16424 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16425 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16426 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16427 // resulting in -
16428 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16429 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16430 %{
16431 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16432 format %{ "convHF2SAndHF2F $dst, $src" %}
16433 ins_encode %{
16434 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16435 %}
16436 ins_pipe(pipe_slow);
16437 %}
16438
16439 // ============================================================================
16440 // This name is KNOWN by the ADLC and cannot be changed.
16441 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16442 // for this guy.
16443 instruct tlsLoadP(thread_RegP dst)
16444 %{
16445 match(Set dst (ThreadLocal));
16446
16447 ins_cost(0);
16448
16449 format %{ " -- \t// $dst=Thread::current(), empty" %}
16450
16451 size(0);
16452
16453 ins_encode( /*empty*/ );
16454
16455 ins_pipe(pipe_class_empty);
16456 %}
16457
16458 //----------PEEPHOLE RULES-----------------------------------------------------
16459 // These must follow all instruction definitions as they use the names
16460 // defined in the instructions definitions.
16461 //
16462 // peepmatch ( root_instr_name [preceding_instruction]* );
16463 //
16464 // peepconstraint %{
16465 // (instruction_number.operand_name relational_op instruction_number.operand_name
16466 // [, ...] );
16467 // // instruction numbers are zero-based using left to right order in peepmatch
16468 //
16469 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16470 // // provide an instruction_number.operand_name for each operand that appears
16471 // // in the replacement instruction's match rule
16472 //
16473 // ---------VM FLAGS---------------------------------------------------------
16474 //
16475 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16476 //
16477 // Each peephole rule is given an identifying number starting with zero and
16478 // increasing by one in the order seen by the parser. An individual peephole
16479 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16480 // on the command-line.
16481 //
16482 // ---------CURRENT LIMITATIONS----------------------------------------------
16483 //
16484 // Only match adjacent instructions in same basic block
16485 // Only equality constraints
16486 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16487 // Only one replacement instruction
16488 //
16489 // ---------EXAMPLE----------------------------------------------------------
16490 //
16491 // // pertinent parts of existing instructions in architecture description
16492 // instruct movI(iRegINoSp dst, iRegI src)
16493 // %{
16494 // match(Set dst (CopyI src));
16495 // %}
16496 //
16497 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16498 // %{
16499 // match(Set dst (AddI dst src));
16500 // effect(KILL cr);
16501 // %}
16502 //
16503 // // Change (inc mov) to lea
16504 // peephole %{
16505 // // increment preceded by register-register move
16506 // peepmatch ( incI_iReg movI );
16507 // // require that the destination register of the increment
16508 // // match the destination register of the move
16509 // peepconstraint ( 0.dst == 1.dst );
16510 // // construct a replacement instruction that sets
16511 // // the destination to ( move's source register + one )
16512 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16513 // %}
16514 //
16515
16516 // Implementation no longer uses movX instructions since
16517 // machine-independent system no longer uses CopyX nodes.
16518 //
16519 // peephole
16520 // %{
16521 // peepmatch (incI_iReg movI);
16522 // peepconstraint (0.dst == 1.dst);
16523 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16524 // %}
16525
16526 // peephole
16527 // %{
16528 // peepmatch (decI_iReg movI);
16529 // peepconstraint (0.dst == 1.dst);
16530 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16531 // %}
16532
16533 // peephole
16534 // %{
16535 // peepmatch (addI_iReg_imm movI);
16536 // peepconstraint (0.dst == 1.dst);
16537 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16538 // %}
16539
16540 // peephole
16541 // %{
16542 // peepmatch (incL_iReg movL);
16543 // peepconstraint (0.dst == 1.dst);
16544 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16545 // %}
16546
16547 // peephole
16548 // %{
16549 // peepmatch (decL_iReg movL);
16550 // peepconstraint (0.dst == 1.dst);
16551 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16552 // %}
16553
16554 // peephole
16555 // %{
16556 // peepmatch (addL_iReg_imm movL);
16557 // peepconstraint (0.dst == 1.dst);
16558 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16559 // %}
16560
16561 // peephole
16562 // %{
16563 // peepmatch (addP_iReg_imm movP);
16564 // peepconstraint (0.dst == 1.dst);
16565 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16566 // %}
16567
16568 // // Change load of spilled value to only a spill
16569 // instruct storeI(memory mem, iRegI src)
16570 // %{
16571 // match(Set mem (StoreI mem src));
16572 // %}
16573 //
16574 // instruct loadI(iRegINoSp dst, memory mem)
16575 // %{
16576 // match(Set dst (LoadI mem));
16577 // %}
16578 //
16579
16580 //----------SMARTSPILL RULES---------------------------------------------------
16581 // These must follow all instruction definitions as they use the names
16582 // defined in the instructions definitions.
16583
16584 // Local Variables:
16585 // mode: c++
16586 // End: