1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else {
1688 return 6 * NativeInstruction::instruction_size;
1689 }
1690 }
1691
1692 //=============================================================================
1693
1694 #ifndef PRODUCT
1695 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1696 st->print("BREAKPOINT");
1697 }
1698 #endif
1699
1700 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1701 __ brk(0);
1702 }
1703
1704 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1705 return MachNode::size(ra_);
1706 }
1707
1708 //=============================================================================
1709
1710 #ifndef PRODUCT
1711 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1712 st->print("nop \t# %d bytes pad for loops and calls", _count);
1713 }
1714 #endif
1715
1716 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1717 for (int i = 0; i < _count; i++) {
1718 __ nop();
1719 }
1720 }
1721
1722 uint MachNopNode::size(PhaseRegAlloc*) const {
1723 return _count * NativeInstruction::instruction_size;
1724 }
1725
1726 //=============================================================================
1727 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1728
1729 int ConstantTable::calculate_table_base_offset() const {
1730 return 0; // absolute addressing, no offset
1731 }
1732
1733 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1734 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1735 ShouldNotReachHere();
1736 }
1737
1738 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1739 // Empty encoding
1740 }
1741
1742 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1743 return 0;
1744 }
1745
1746 #ifndef PRODUCT
1747 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1748 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1749 }
1750 #endif
1751
1752 #ifndef PRODUCT
1753 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1754 Compile* C = ra_->C;
1755
1756 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1757
1758 if (C->output()->need_stack_bang(framesize))
1759 st->print("# stack bang size=%d\n\t", framesize);
1760
1761 if (VM_Version::use_rop_protection()) {
1762 st->print("ldr zr, [lr]\n\t");
1763 st->print("paciaz\n\t");
1764 }
1765 if (framesize < ((1 << 9) + 2 * wordSize)) {
1766 st->print("sub sp, sp, #%d\n\t", framesize);
1767 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1768 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1769 } else {
1770 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1771 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1772 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1773 st->print("sub sp, sp, rscratch1");
1774 }
1775 if (C->stub_function() == nullptr) {
1776 st->print("\n\t");
1777 st->print("ldr rscratch1, [guard]\n\t");
1778 st->print("dmb ishld\n\t");
1779 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1780 st->print("cmp rscratch1, rscratch2\n\t");
1781 st->print("b.eq skip");
1782 st->print("\n\t");
1783 st->print("blr #nmethod_entry_barrier_stub\n\t");
1784 st->print("b skip\n\t");
1785 st->print("guard: int\n\t");
1786 st->print("\n\t");
1787 st->print("skip:\n\t");
1788 }
1789 }
1790 #endif
1791
1792 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1793 Compile* C = ra_->C;
1794
1795 // n.b. frame size includes space for return pc and rfp
1796 const int framesize = C->output()->frame_size_in_bytes();
1797
1798 if (C->clinit_barrier_on_entry()) {
1799 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1800
1801 Label L_skip_barrier;
1802
1803 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1804 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1805 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1806 __ bind(L_skip_barrier);
1807 }
1808
1809 if (C->max_vector_size() > 0) {
1810 __ reinitialize_ptrue();
1811 }
1812
1813 int bangsize = C->output()->bang_size_in_bytes();
1814 if (C->output()->need_stack_bang(bangsize))
1815 __ generate_stack_overflow_check(bangsize);
1816
1817 __ build_frame(framesize);
1818
1819 if (C->stub_function() == nullptr) {
1820 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1821 // Dummy labels for just measuring the code size
1822 Label dummy_slow_path;
1823 Label dummy_continuation;
1824 Label dummy_guard;
1825 Label* slow_path = &dummy_slow_path;
1826 Label* continuation = &dummy_continuation;
1827 Label* guard = &dummy_guard;
1828 if (!Compile::current()->output()->in_scratch_emit_size()) {
1829 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1830 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1831 Compile::current()->output()->add_stub(stub);
1832 slow_path = &stub->entry();
1833 continuation = &stub->continuation();
1834 guard = &stub->guard();
1835 }
1836 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1837 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1838 }
1839
1840 if (VerifyStackAtCalls) {
1841 Unimplemented();
1842 }
1843
1844 C->output()->set_frame_complete(__ offset());
1845
1846 if (C->has_mach_constant_base_node()) {
1847 // NOTE: We set the table base offset here because users might be
1848 // emitted before MachConstantBaseNode.
1849 ConstantTable& constant_table = C->output()->constant_table();
1850 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1851 }
1852 }
1853
1854 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1855 {
1856 return MachNode::size(ra_); // too many variables; just compute it
1857 // the hard way
1858 }
1859
1860 int MachPrologNode::reloc() const
1861 {
1862 return 0;
1863 }
1864
1865 //=============================================================================
1866
1867 #ifndef PRODUCT
1868 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1869 Compile* C = ra_->C;
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 st->print("# pop frame %d\n\t",framesize);
1873
1874 if (framesize == 0) {
1875 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1876 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1877 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1878 st->print("add sp, sp, #%d\n\t", framesize);
1879 } else {
1880 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1881 st->print("add sp, sp, rscratch1\n\t");
1882 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1883 }
1884 if (VM_Version::use_rop_protection()) {
1885 st->print("autiaz\n\t");
1886 st->print("ldr zr, [lr]\n\t");
1887 }
1888
1889 if (do_polling() && C->is_method_compilation()) {
1890 st->print("# test polling word\n\t");
1891 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1892 st->print("cmp sp, rscratch1\n\t");
1893 st->print("bhi #slow_path");
1894 }
1895 }
1896 #endif
1897
1898 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1899 Compile* C = ra_->C;
1900 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1901
1902 __ remove_frame(framesize);
1903
1904 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1905 __ reserved_stack_check();
1906 }
1907
1908 if (do_polling() && C->is_method_compilation()) {
1909 Label dummy_label;
1910 Label* code_stub = &dummy_label;
1911 if (!C->output()->in_scratch_emit_size()) {
1912 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1913 C->output()->add_stub(stub);
1914 code_stub = &stub->entry();
1915 }
1916 __ relocate(relocInfo::poll_return_type);
1917 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1918 }
1919 }
1920
1921 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1922 // Variable size. Determine dynamically.
1923 return MachNode::size(ra_);
1924 }
1925
1926 int MachEpilogNode::reloc() const {
1927 // Return number of relocatable values contained in this instruction.
1928 return 1; // 1 for polling page.
1929 }
1930
1931 const Pipeline * MachEpilogNode::pipeline() const {
1932 return MachNode::pipeline_class();
1933 }
1934
1935 //=============================================================================
1936
1937 static enum RC rc_class(OptoReg::Name reg) {
1938
1939 if (reg == OptoReg::Bad) {
1940 return rc_bad;
1941 }
1942
1943 // we have 32 int registers * 2 halves
1944 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1945
1946 if (reg < slots_of_int_registers) {
1947 return rc_int;
1948 }
1949
1950 // we have 32 float register * 8 halves
1951 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1952 if (reg < slots_of_int_registers + slots_of_float_registers) {
1953 return rc_float;
1954 }
1955
1956 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1957 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1958 return rc_predicate;
1959 }
1960
1961 // Between predicate regs & stack is the flags.
1962 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1963
1964 return rc_stack;
1965 }
1966
1967 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1968 Compile* C = ra_->C;
1969
1970 // Get registers to move.
1971 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1972 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1973 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1974 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1975
1976 enum RC src_hi_rc = rc_class(src_hi);
1977 enum RC src_lo_rc = rc_class(src_lo);
1978 enum RC dst_hi_rc = rc_class(dst_hi);
1979 enum RC dst_lo_rc = rc_class(dst_lo);
1980
1981 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1982
1983 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1984 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1985 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1986 "expected aligned-adjacent pairs");
1987 }
1988
1989 if (src_lo == dst_lo && src_hi == dst_hi) {
1990 return 0; // Self copy, no move.
1991 }
1992
1993 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1994 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1995 int src_offset = ra_->reg2offset(src_lo);
1996 int dst_offset = ra_->reg2offset(dst_lo);
1997
1998 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1999 uint ireg = ideal_reg();
2000 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2001 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2002 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2003 if (ireg == Op_VecA && masm) {
2004 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2005 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2006 // stack->stack
2007 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2008 sve_vector_reg_size_in_bytes);
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2013 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2016 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]),
2018 as_FloatRegister(Matcher::_regEncode[src_lo]));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 } else if (masm) {
2023 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2024 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2025 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2026 // stack->stack
2027 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2028 if (ireg == Op_VecD) {
2029 __ unspill(rscratch1, true, src_offset);
2030 __ spill(rscratch1, true, dst_offset);
2031 } else {
2032 __ spill_copy128(src_offset, dst_offset);
2033 }
2034 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2035 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2036 ireg == Op_VecD ? __ T8B : __ T16B,
2037 as_FloatRegister(Matcher::_regEncode[src_lo]));
2038 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2039 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2040 ireg == Op_VecD ? __ D : __ Q,
2041 ra_->reg2offset(dst_lo));
2042 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2043 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 ireg == Op_VecD ? __ D : __ Q,
2045 ra_->reg2offset(src_lo));
2046 } else {
2047 ShouldNotReachHere();
2048 }
2049 }
2050 } else if (masm) {
2051 switch (src_lo_rc) {
2052 case rc_int:
2053 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2054 if (is64) {
2055 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_Register(Matcher::_regEncode[src_lo]));
2057 } else {
2058 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 }
2061 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2062 if (is64) {
2063 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_Register(Matcher::_regEncode[src_lo]));
2065 } else {
2066 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 }
2069 } else { // gpr --> stack spill
2070 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2071 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2072 }
2073 break;
2074 case rc_float:
2075 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2076 if (is64) {
2077 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2078 as_FloatRegister(Matcher::_regEncode[src_lo]));
2079 } else {
2080 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 }
2083 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2084 if (is64) {
2085 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2086 as_FloatRegister(Matcher::_regEncode[src_lo]));
2087 } else {
2088 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 }
2091 } else { // fpr --> stack spill
2092 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2093 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2094 is64 ? __ D : __ S, dst_offset);
2095 }
2096 break;
2097 case rc_stack:
2098 if (dst_lo_rc == rc_int) { // stack --> gpr load
2099 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2100 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2101 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2102 is64 ? __ D : __ S, src_offset);
2103 } else if (dst_lo_rc == rc_predicate) {
2104 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2105 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2106 } else { // stack --> stack copy
2107 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2108 if (ideal_reg() == Op_RegVectMask) {
2109 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2110 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2111 } else {
2112 __ unspill(rscratch1, is64, src_offset);
2113 __ spill(rscratch1, is64, dst_offset);
2114 }
2115 }
2116 break;
2117 case rc_predicate:
2118 if (dst_lo_rc == rc_predicate) {
2119 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2120 } else if (dst_lo_rc == rc_stack) {
2121 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2122 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2123 } else {
2124 assert(false, "bad src and dst rc_class combination.");
2125 ShouldNotReachHere();
2126 }
2127 break;
2128 default:
2129 assert(false, "bad rc_class for spill");
2130 ShouldNotReachHere();
2131 }
2132 }
2133
2134 if (st) {
2135 st->print("spill ");
2136 if (src_lo_rc == rc_stack) {
2137 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2138 } else {
2139 st->print("%s -> ", Matcher::regName[src_lo]);
2140 }
2141 if (dst_lo_rc == rc_stack) {
2142 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2143 } else {
2144 st->print("%s", Matcher::regName[dst_lo]);
2145 }
2146 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2147 int vsize = 0;
2148 switch (ideal_reg()) {
2149 case Op_VecD:
2150 vsize = 64;
2151 break;
2152 case Op_VecX:
2153 vsize = 128;
2154 break;
2155 case Op_VecA:
2156 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2157 break;
2158 default:
2159 assert(false, "bad register type for spill");
2160 ShouldNotReachHere();
2161 }
2162 st->print("\t# vector spill size = %d", vsize);
2163 } else if (ideal_reg() == Op_RegVectMask) {
2164 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2165 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2166 st->print("\t# predicate spill size = %d", vsize);
2167 } else {
2168 st->print("\t# spill size = %d", is64 ? 64 : 32);
2169 }
2170 }
2171
2172 return 0;
2173
2174 }
2175
2176 #ifndef PRODUCT
2177 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2178 if (!ra_)
2179 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2180 else
2181 implementation(nullptr, ra_, false, st);
2182 }
2183 #endif
2184
2185 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2186 implementation(masm, ra_, false, nullptr);
2187 }
2188
2189 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2190 return MachNode::size(ra_);
2191 }
2192
2193 //=============================================================================
2194
2195 #ifndef PRODUCT
2196 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2197 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2198 int reg = ra_->get_reg_first(this);
2199 st->print("add %s, rsp, #%d]\t# box lock",
2200 Matcher::regName[reg], offset);
2201 }
2202 #endif
2203
2204 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2205 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2206 int reg = ra_->get_encode(this);
2207
2208 // This add will handle any 24-bit signed offset. 24 bits allows an
2209 // 8 megabyte stack frame.
2210 __ add(as_Register(reg), sp, offset);
2211 }
2212
2213 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2214 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2215 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2216
2217 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2218 return NativeInstruction::instruction_size;
2219 } else {
2220 return 2 * NativeInstruction::instruction_size;
2221 }
2222 }
2223
2224 //=============================================================================
2225
2226 #ifndef PRODUCT
2227 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2228 {
2229 st->print_cr("# MachUEPNode");
2230 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2231 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2232 st->print_cr("\tcmpw rscratch1, r10");
2233 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2234 }
2235 #endif
2236
2237 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2238 {
2239 __ ic_check(InteriorEntryAlignment);
2240 }
2241
2242 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2243 {
2244 return MachNode::size(ra_);
2245 }
2246
2247 // REQUIRED EMIT CODE
2248
2249 //=============================================================================
2250
2251 // Emit deopt handler code.
2252 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2253 {
2254 // Note that the code buffer's insts_mark is always relative to insts.
2255 // That's why we must use the macroassembler to generate a handler.
2256 address base = __ start_a_stub(size_deopt_handler());
2257 if (base == nullptr) {
2258 ciEnv::current()->record_failure("CodeCache is full");
2259 return 0; // CodeBuffer::expand failed
2260 }
2261
2262 int offset = __ offset();
2263 Label start;
2264 __ bind(start);
2265 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2266
2267 int entry_offset = __ offset();
2268 __ b(start);
2269
2270 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2271 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2272 "out of bounds read in post-call NOP check");
2273 __ end_a_stub();
2274 return entry_offset;
2275 }
2276
2277 // REQUIRED MATCHER CODE
2278
2279 //=============================================================================
2280
2281 bool Matcher::match_rule_supported(int opcode) {
2282 if (!has_match_rule(opcode))
2283 return false;
2284
2285 switch (opcode) {
2286 case Op_OnSpinWait:
2287 return VM_Version::supports_on_spin_wait();
2288 case Op_CacheWB:
2289 case Op_CacheWBPreSync:
2290 case Op_CacheWBPostSync:
2291 if (!VM_Version::supports_data_cache_line_flush()) {
2292 return false;
2293 }
2294 break;
2295 case Op_ExpandBits:
2296 case Op_CompressBits:
2297 if (!VM_Version::supports_svebitperm()) {
2298 return false;
2299 }
2300 break;
2301 case Op_FmaF:
2302 case Op_FmaD:
2303 case Op_FmaVF:
2304 case Op_FmaVD:
2305 if (!UseFMA) {
2306 return false;
2307 }
2308 break;
2309 case Op_FmaHF:
2310 // UseFMA flag also needs to be checked along with FEAT_FP16
2311 if (!UseFMA || !is_feat_fp16_supported()) {
2312 return false;
2313 }
2314 break;
2315 case Op_AddHF:
2316 case Op_SubHF:
2317 case Op_MulHF:
2318 case Op_DivHF:
2319 case Op_MinHF:
2320 case Op_MaxHF:
2321 case Op_SqrtHF:
2322 // Half-precision floating point scalar operations require FEAT_FP16
2323 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2324 // features are supported.
2325 if (!is_feat_fp16_supported()) {
2326 return false;
2327 }
2328 break;
2329 }
2330
2331 return true; // Per default match rules are supported.
2332 }
2333
2334 const RegMask* Matcher::predicate_reg_mask(void) {
2335 return &_PR_REG_mask;
2336 }
2337
2338 bool Matcher::supports_vector_calling_convention(void) {
2339 return EnableVectorSupport;
2340 }
2341
2342 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2343 assert(EnableVectorSupport, "sanity");
2344 int lo = V0_num;
2345 int hi = V0_H_num;
2346 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2347 hi = V0_K_num;
2348 }
2349 return OptoRegPair(hi, lo);
2350 }
2351
2352 // Is this branch offset short enough that a short branch can be used?
2353 //
2354 // NOTE: If the platform does not provide any short branch variants, then
2355 // this method should return false for offset 0.
2356 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2357 // The passed offset is relative to address of the branch.
2358
2359 return (-32768 <= offset && offset < 32768);
2360 }
2361
2362 // Vector width in bytes.
2363 int Matcher::vector_width_in_bytes(BasicType bt) {
2364 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2365 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2366 // Minimum 2 values in vector
2367 if (size < 2*type2aelembytes(bt)) size = 0;
2368 // But never < 4
2369 if (size < 4) size = 0;
2370 return size;
2371 }
2372
2373 // Limits on vector size (number of elements) loaded into vector.
2374 int Matcher::max_vector_size(const BasicType bt) {
2375 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2376 }
2377
2378 int Matcher::min_vector_size(const BasicType bt) {
2379 // Usually, the shortest vector length supported by AArch64 ISA and
2380 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2381 // vectors in a few special cases.
2382 int size;
2383 switch(bt) {
2384 case T_BOOLEAN:
2385 // Load/store a vector mask with only 2 elements for vector types
2386 // such as "2I/2F/2L/2D".
2387 size = 2;
2388 break;
2389 case T_BYTE:
2390 // Generate a "4B" vector, to support vector cast between "8B/16B"
2391 // and "4S/4I/4L/4F/4D".
2392 size = 4;
2393 break;
2394 case T_SHORT:
2395 // Generate a "2S" vector, to support vector cast between "4S/8S"
2396 // and "2I/2L/2F/2D".
2397 size = 2;
2398 break;
2399 default:
2400 // Limit the min vector length to 64-bit.
2401 size = 8 / type2aelembytes(bt);
2402 // The number of elements in a vector should be at least 2.
2403 size = MAX2(size, 2);
2404 }
2405
2406 int max_size = max_vector_size(bt);
2407 return MIN2(size, max_size);
2408 }
2409
2410 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2411 return Matcher::max_vector_size(bt);
2412 }
2413
2414 // Actual max scalable vector register length.
2415 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2416 return Matcher::max_vector_size(bt);
2417 }
2418
2419 // Vector ideal reg.
2420 uint Matcher::vector_ideal_reg(int len) {
2421 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2422 return Op_VecA;
2423 }
2424 switch(len) {
2425 // For 16-bit/32-bit mask vector, reuse VecD.
2426 case 2:
2427 case 4:
2428 case 8: return Op_VecD;
2429 case 16: return Op_VecX;
2430 }
2431 ShouldNotReachHere();
2432 return 0;
2433 }
2434
2435 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2436 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2437 switch (ideal_reg) {
2438 case Op_VecA: return new vecAOper();
2439 case Op_VecD: return new vecDOper();
2440 case Op_VecX: return new vecXOper();
2441 }
2442 ShouldNotReachHere();
2443 return nullptr;
2444 }
2445
2446 bool Matcher::is_reg2reg_move(MachNode* m) {
2447 return false;
2448 }
2449
2450 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2451 return false;
2452 }
2453
2454 bool Matcher::is_generic_vector(MachOper* opnd) {
2455 return opnd->opcode() == VREG;
2456 }
2457
2458 #ifdef ASSERT
2459 // Return whether or not this register is ever used as an argument.
2460 bool Matcher::can_be_java_arg(int reg)
2461 {
2462 return
2463 reg == R0_num || reg == R0_H_num ||
2464 reg == R1_num || reg == R1_H_num ||
2465 reg == R2_num || reg == R2_H_num ||
2466 reg == R3_num || reg == R3_H_num ||
2467 reg == R4_num || reg == R4_H_num ||
2468 reg == R5_num || reg == R5_H_num ||
2469 reg == R6_num || reg == R6_H_num ||
2470 reg == R7_num || reg == R7_H_num ||
2471 reg == V0_num || reg == V0_H_num ||
2472 reg == V1_num || reg == V1_H_num ||
2473 reg == V2_num || reg == V2_H_num ||
2474 reg == V3_num || reg == V3_H_num ||
2475 reg == V4_num || reg == V4_H_num ||
2476 reg == V5_num || reg == V5_H_num ||
2477 reg == V6_num || reg == V6_H_num ||
2478 reg == V7_num || reg == V7_H_num;
2479 }
2480 #endif
2481
2482 uint Matcher::int_pressure_limit()
2483 {
2484 // JDK-8183543: When taking the number of available registers as int
2485 // register pressure threshold, the jtreg test:
2486 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2487 // failed due to C2 compilation failure with
2488 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2489 //
2490 // A derived pointer is live at CallNode and then is flagged by RA
2491 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2492 // derived pointers and lastly fail to spill after reaching maximum
2493 // number of iterations. Lowering the default pressure threshold to
2494 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2495 // a high register pressure area of the code so that split_DEF can
2496 // generate DefinitionSpillCopy for the derived pointer.
2497 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2498 if (!PreserveFramePointer) {
2499 // When PreserveFramePointer is off, frame pointer is allocatable,
2500 // but different from other SOC registers, it is excluded from
2501 // fatproj's mask because its save type is No-Save. Decrease 1 to
2502 // ensure high pressure at fatproj when PreserveFramePointer is off.
2503 // See check_pressure_at_fatproj().
2504 default_int_pressure_threshold--;
2505 }
2506 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2507 }
2508
2509 uint Matcher::float_pressure_limit()
2510 {
2511 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2512 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2513 }
2514
2515 const RegMask& Matcher::divI_proj_mask() {
2516 ShouldNotReachHere();
2517 return RegMask::EMPTY;
2518 }
2519
2520 // Register for MODI projection of divmodI.
2521 const RegMask& Matcher::modI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for DIVL projection of divmodL.
2527 const RegMask& Matcher::divL_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for MODL projection of divmodL.
2533 const RegMask& Matcher::modL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2539 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2540 Node* u = addp->fast_out(i);
2541 if (u->is_LoadStore()) {
2542 // On AArch64, LoadStoreNodes (i.e. compare and swap
2543 // instructions) only take register indirect as an operand, so
2544 // any attempt to use an AddPNode as an input to a LoadStoreNode
2545 // must fail.
2546 return false;
2547 }
2548 if (u->is_Mem()) {
2549 int opsize = u->as_Mem()->memory_size();
2550 assert(opsize > 0, "unexpected memory operand size");
2551 if (u->as_Mem()->memory_size() != (1<<shift)) {
2552 return false;
2553 }
2554 }
2555 }
2556 return true;
2557 }
2558
2559 // Convert BoolTest condition to Assembler condition.
2560 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2561 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2562 Assembler::Condition result;
2563 switch(cond) {
2564 case BoolTest::eq:
2565 result = Assembler::EQ; break;
2566 case BoolTest::ne:
2567 result = Assembler::NE; break;
2568 case BoolTest::le:
2569 result = Assembler::LE; break;
2570 case BoolTest::ge:
2571 result = Assembler::GE; break;
2572 case BoolTest::lt:
2573 result = Assembler::LT; break;
2574 case BoolTest::gt:
2575 result = Assembler::GT; break;
2576 case BoolTest::ule:
2577 result = Assembler::LS; break;
2578 case BoolTest::uge:
2579 result = Assembler::HS; break;
2580 case BoolTest::ult:
2581 result = Assembler::LO; break;
2582 case BoolTest::ugt:
2583 result = Assembler::HI; break;
2584 case BoolTest::overflow:
2585 result = Assembler::VS; break;
2586 case BoolTest::no_overflow:
2587 result = Assembler::VC; break;
2588 default:
2589 ShouldNotReachHere();
2590 return Assembler::Condition(-1);
2591 }
2592
2593 // Check conversion
2594 if (cond & BoolTest::unsigned_compare) {
2595 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2596 } else {
2597 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2598 }
2599
2600 return result;
2601 }
2602
2603 // Binary src (Replicate con)
2604 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2605 if (n == nullptr || m == nullptr) {
2606 return false;
2607 }
2608
2609 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2610 return false;
2611 }
2612
2613 Node* imm_node = m->in(1);
2614 if (!imm_node->is_Con()) {
2615 return false;
2616 }
2617
2618 const Type* t = imm_node->bottom_type();
2619 if (!(t->isa_int() || t->isa_long())) {
2620 return false;
2621 }
2622
2623 switch (n->Opcode()) {
2624 case Op_AndV:
2625 case Op_OrV:
2626 case Op_XorV: {
2627 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2628 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2629 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2630 }
2631 case Op_AddVB:
2632 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2633 case Op_AddVS:
2634 case Op_AddVI:
2635 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2636 case Op_AddVL:
2637 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2638 default:
2639 return false;
2640 }
2641 }
2642
2643 // (XorV src (Replicate m1))
2644 // (XorVMask src (MaskAll m1))
2645 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2646 if (n != nullptr && m != nullptr) {
2647 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2648 VectorNode::is_all_ones_vector(m);
2649 }
2650 return false;
2651 }
2652
2653 // Should the matcher clone input 'm' of node 'n'?
2654 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2655 if (is_vshift_con_pattern(n, m) ||
2656 is_vector_bitwise_not_pattern(n, m) ||
2657 is_valid_sve_arith_imm_pattern(n, m) ||
2658 is_encode_and_store_pattern(n, m)) {
2659 mstack.push(m, Visit);
2660 return true;
2661 }
2662 return false;
2663 }
2664
2665 // Should the Matcher clone shifts on addressing modes, expecting them
2666 // to be subsumed into complex addressing expressions or compute them
2667 // into registers?
2668 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2669
2670 // Loads and stores with indirect memory input (e.g., volatile loads and
2671 // stores) do not subsume the input into complex addressing expressions. If
2672 // the addressing expression is input to at least one such load or store, do
2673 // not clone the addressing expression. Query needs_acquiring_load and
2674 // needs_releasing_store as a proxy for indirect memory input, as it is not
2675 // possible to directly query for indirect memory input at this stage.
2676 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2677 Node* n = m->fast_out(i);
2678 if (n->is_Load() && needs_acquiring_load(n)) {
2679 return false;
2680 }
2681 if (n->is_Store() && needs_releasing_store(n)) {
2682 return false;
2683 }
2684 }
2685
2686 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2687 return true;
2688 }
2689
2690 Node *off = m->in(AddPNode::Offset);
2691 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2692 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2693 // Are there other uses besides address expressions?
2694 !is_visited(off)) {
2695 address_visited.set(off->_idx); // Flag as address_visited
2696 mstack.push(off->in(2), Visit);
2697 Node *conv = off->in(1);
2698 if (conv->Opcode() == Op_ConvI2L &&
2699 // Are there other uses besides address expressions?
2700 !is_visited(conv)) {
2701 address_visited.set(conv->_idx); // Flag as address_visited
2702 mstack.push(conv->in(1), Pre_Visit);
2703 } else {
2704 mstack.push(conv, Pre_Visit);
2705 }
2706 address_visited.test_set(m->_idx); // Flag as address_visited
2707 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2708 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2709 return true;
2710 } else if (off->Opcode() == Op_ConvI2L &&
2711 // Are there other uses besides address expressions?
2712 !is_visited(off)) {
2713 address_visited.test_set(m->_idx); // Flag as address_visited
2714 address_visited.set(off->_idx); // Flag as address_visited
2715 mstack.push(off->in(1), Pre_Visit);
2716 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2717 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2718 return true;
2719 }
2720 return false;
2721 }
2722
2723 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2724 { \
2725 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2726 guarantee(DISP == 0, "mode not permitted for volatile"); \
2727 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2728 __ INSN(REG, as_Register(BASE)); \
2729 }
2730
2731
2732 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2733 {
2734 Address::extend scale;
2735
2736 // Hooboy, this is fugly. We need a way to communicate to the
2737 // encoder that the index needs to be sign extended, so we have to
2738 // enumerate all the cases.
2739 switch (opcode) {
2740 case INDINDEXSCALEDI2L:
2741 case INDINDEXSCALEDI2LN:
2742 case INDINDEXI2L:
2743 case INDINDEXI2LN:
2744 scale = Address::sxtw(size);
2745 break;
2746 default:
2747 scale = Address::lsl(size);
2748 }
2749
2750 if (index == -1) {
2751 return Address(base, disp);
2752 } else {
2753 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2754 return Address(base, as_Register(index), scale);
2755 }
2756 }
2757
2758
2759 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2760 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2761 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2762 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2763 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2764
2765 // Used for all non-volatile memory accesses. The use of
2766 // $mem->opcode() to discover whether this pattern uses sign-extended
2767 // offsets is something of a kludge.
2768 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2769 Register reg, int opcode,
2770 Register base, int index, int scale, int disp,
2771 int size_in_memory)
2772 {
2773 Address addr = mem2address(opcode, base, index, scale, disp);
2774 if (addr.getMode() == Address::base_plus_offset) {
2775 /* Fix up any out-of-range offsets. */
2776 assert_different_registers(rscratch1, base);
2777 assert_different_registers(rscratch1, reg);
2778 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2779 }
2780 (masm->*insn)(reg, addr);
2781 }
2782
2783 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2784 FloatRegister reg, int opcode,
2785 Register base, int index, int size, int disp,
2786 int size_in_memory)
2787 {
2788 Address::extend scale;
2789
2790 switch (opcode) {
2791 case INDINDEXSCALEDI2L:
2792 case INDINDEXSCALEDI2LN:
2793 scale = Address::sxtw(size);
2794 break;
2795 default:
2796 scale = Address::lsl(size);
2797 }
2798
2799 if (index == -1) {
2800 // Fix up any out-of-range offsets.
2801 assert_different_registers(rscratch1, base);
2802 Address addr = Address(base, disp);
2803 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2804 (masm->*insn)(reg, addr);
2805 } else {
2806 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2807 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2808 }
2809 }
2810
2811 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2812 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2813 int opcode, Register base, int index, int size, int disp)
2814 {
2815 if (index == -1) {
2816 (masm->*insn)(reg, T, Address(base, disp));
2817 } else {
2818 assert(disp == 0, "unsupported address mode");
2819 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2820 }
2821 }
2822
2823 %}
2824
2825
2826
2827 //----------ENCODING BLOCK-----------------------------------------------------
2828 // This block specifies the encoding classes used by the compiler to
2829 // output byte streams. Encoding classes are parameterized macros
2830 // used by Machine Instruction Nodes in order to generate the bit
2831 // encoding of the instruction. Operands specify their base encoding
2832 // interface with the interface keyword. There are currently
2833 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2834 // COND_INTER. REG_INTER causes an operand to generate a function
2835 // which returns its register number when queried. CONST_INTER causes
2836 // an operand to generate a function which returns the value of the
2837 // constant when queried. MEMORY_INTER causes an operand to generate
2838 // four functions which return the Base Register, the Index Register,
2839 // the Scale Value, and the Offset Value of the operand when queried.
2840 // COND_INTER causes an operand to generate six functions which return
2841 // the encoding code (ie - encoding bits for the instruction)
2842 // associated with each basic boolean condition for a conditional
2843 // instruction.
2844 //
2845 // Instructions specify two basic values for encoding. Again, a
2846 // function is available to check if the constant displacement is an
2847 // oop. They use the ins_encode keyword to specify their encoding
2848 // classes (which must be a sequence of enc_class names, and their
2849 // parameters, specified in the encoding block), and they use the
2850 // opcode keyword to specify, in order, their primary, secondary, and
2851 // tertiary opcode. Only the opcode sections which a particular
2852 // instruction needs for encoding need to be specified.
2853 encode %{
2854 // Build emit functions for each basic byte or larger field in the
2855 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2856 // from C++ code in the enc_class source block. Emit functions will
2857 // live in the main source block for now. In future, we can
2858 // generalize this by adding a syntax that specifies the sizes of
2859 // fields in an order, so that the adlc can build the emit functions
2860 // automagically
2861
2862 // catch all for unimplemented encodings
2863 enc_class enc_unimplemented %{
2864 __ unimplemented("C2 catch all");
2865 %}
2866
2867 // BEGIN Non-volatile memory access
2868
2869 // This encoding class is generated automatically from ad_encode.m4.
2870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2871 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2872 Register dst_reg = as_Register($dst$$reg);
2873 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2874 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2875 %}
2876
2877 // This encoding class is generated automatically from ad_encode.m4.
2878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2879 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2880 Register dst_reg = as_Register($dst$$reg);
2881 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2883 %}
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2968 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2976 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2984 Register src_reg = as_Register($src$$reg);
2985 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_strb0(memory1 mem) %{
2992 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2993 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2994 %}
2995
2996 // This encoding class is generated automatically from ad_encode.m4.
2997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2998 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2999 Register src_reg = as_Register($src$$reg);
3000 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3001 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3002 %}
3003
3004 // This encoding class is generated automatically from ad_encode.m4.
3005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3006 enc_class aarch64_enc_strh0(memory2 mem) %{
3007 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3008 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3009 %}
3010
3011 // This encoding class is generated automatically from ad_encode.m4.
3012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3013 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3014 Register src_reg = as_Register($src$$reg);
3015 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strw0(memory4 mem) %{
3022 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3029 Register src_reg = as_Register($src$$reg);
3030 // we sometimes get asked to store the stack pointer into the
3031 // current thread -- we cannot do that directly on AArch64
3032 if (src_reg == r31_sp) {
3033 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3034 __ mov(rscratch2, sp);
3035 src_reg = rscratch2;
3036 }
3037 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_str0(memory8 mem) %{
3044 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3051 FloatRegister src_reg = as_FloatRegister($src$$reg);
3052 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3054 %}
3055
3056 // This encoding class is generated automatically from ad_encode.m4.
3057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3058 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3059 FloatRegister src_reg = as_FloatRegister($src$$reg);
3060 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3067 __ membar(Assembler::StoreStore);
3068 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3069 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3070 %}
3071
3072 // END Non-volatile memory access
3073
3074 // Vector loads and stores
3075 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3076 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3077 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3078 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3079 %}
3080
3081 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3082 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3083 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3084 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3085 %}
3086
3087 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3088 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3089 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3090 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3091 %}
3092
3093 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3100 FloatRegister src_reg = as_FloatRegister($src$$reg);
3101 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3106 FloatRegister src_reg = as_FloatRegister($src$$reg);
3107 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3112 FloatRegister src_reg = as_FloatRegister($src$$reg);
3113 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 // volatile loads and stores
3124
3125 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3126 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3127 rscratch1, stlrb);
3128 %}
3129
3130 enc_class aarch64_enc_stlrb0(memory mem) %{
3131 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3132 rscratch1, stlrb);
3133 %}
3134
3135 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3136 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, stlrh);
3138 %}
3139
3140 enc_class aarch64_enc_stlrh0(memory mem) %{
3141 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, stlrh);
3143 %}
3144
3145 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3146 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, stlrw);
3148 %}
3149
3150 enc_class aarch64_enc_stlrw0(memory mem) %{
3151 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3152 rscratch1, stlrw);
3153 %}
3154
3155 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3156 Register dst_reg = as_Register($dst$$reg);
3157 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, ldarb);
3159 __ sxtbw(dst_reg, dst_reg);
3160 %}
3161
3162 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3163 Register dst_reg = as_Register($dst$$reg);
3164 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, ldarb);
3166 __ sxtb(dst_reg, dst_reg);
3167 %}
3168
3169 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarb);
3172 %}
3173
3174 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarb);
3177 %}
3178
3179 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3180 Register dst_reg = as_Register($dst$$reg);
3181 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarh);
3183 __ sxthw(dst_reg, dst_reg);
3184 %}
3185
3186 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3187 Register dst_reg = as_Register($dst$$reg);
3188 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarh);
3190 __ sxth(dst_reg, dst_reg);
3191 %}
3192
3193 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3194 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarh);
3196 %}
3197
3198 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3199 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarh);
3201 %}
3202
3203 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3204 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarw);
3206 %}
3207
3208 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3209 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldarw);
3211 %}
3212
3213 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3214 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3215 rscratch1, ldar);
3216 %}
3217
3218 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3219 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3220 rscratch1, ldarw);
3221 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3222 %}
3223
3224 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3225 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldar);
3227 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3228 %}
3229
3230 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3231 Register src_reg = as_Register($src$$reg);
3232 // we sometimes get asked to store the stack pointer into the
3233 // current thread -- we cannot do that directly on AArch64
3234 if (src_reg == r31_sp) {
3235 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3236 __ mov(rscratch2, sp);
3237 src_reg = rscratch2;
3238 }
3239 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3240 rscratch1, stlr);
3241 %}
3242
3243 enc_class aarch64_enc_stlr0(memory mem) %{
3244 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3245 rscratch1, stlr);
3246 %}
3247
3248 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3249 {
3250 FloatRegister src_reg = as_FloatRegister($src$$reg);
3251 __ fmovs(rscratch2, src_reg);
3252 }
3253 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3254 rscratch1, stlrw);
3255 %}
3256
3257 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3258 {
3259 FloatRegister src_reg = as_FloatRegister($src$$reg);
3260 __ fmovd(rscratch2, src_reg);
3261 }
3262 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, stlr);
3264 %}
3265
3266 // synchronized read/update encodings
3267
3268 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3269 Register dst_reg = as_Register($dst$$reg);
3270 Register base = as_Register($mem$$base);
3271 int index = $mem$$index;
3272 int scale = $mem$$scale;
3273 int disp = $mem$$disp;
3274 if (index == -1) {
3275 if (disp != 0) {
3276 __ lea(rscratch1, Address(base, disp));
3277 __ ldaxr(dst_reg, rscratch1);
3278 } else {
3279 // TODO
3280 // should we ever get anything other than this case?
3281 __ ldaxr(dst_reg, base);
3282 }
3283 } else {
3284 Register index_reg = as_Register(index);
3285 if (disp == 0) {
3286 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3287 __ ldaxr(dst_reg, rscratch1);
3288 } else {
3289 __ lea(rscratch1, Address(base, disp));
3290 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3291 __ ldaxr(dst_reg, rscratch1);
3292 }
3293 }
3294 %}
3295
3296 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3297 Register src_reg = as_Register($src$$reg);
3298 Register base = as_Register($mem$$base);
3299 int index = $mem$$index;
3300 int scale = $mem$$scale;
3301 int disp = $mem$$disp;
3302 if (index == -1) {
3303 if (disp != 0) {
3304 __ lea(rscratch2, Address(base, disp));
3305 __ stlxr(rscratch1, src_reg, rscratch2);
3306 } else {
3307 // TODO
3308 // should we ever get anything other than this case?
3309 __ stlxr(rscratch1, src_reg, base);
3310 }
3311 } else {
3312 Register index_reg = as_Register(index);
3313 if (disp == 0) {
3314 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3315 __ stlxr(rscratch1, src_reg, rscratch2);
3316 } else {
3317 __ lea(rscratch2, Address(base, disp));
3318 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3319 __ stlxr(rscratch1, src_reg, rscratch2);
3320 }
3321 }
3322 __ cmpw(rscratch1, zr);
3323 %}
3324
3325 // prefetch encodings
3326
3327 enc_class aarch64_enc_prefetchw(memory mem) %{
3328 Register base = as_Register($mem$$base);
3329 int index = $mem$$index;
3330 int scale = $mem$$scale;
3331 int disp = $mem$$disp;
3332 if (index == -1) {
3333 // Fix up any out-of-range offsets.
3334 assert_different_registers(rscratch1, base);
3335 Address addr = Address(base, disp);
3336 addr = __ legitimize_address(addr, 8, rscratch1);
3337 __ prfm(addr, PSTL1KEEP);
3338 } else {
3339 Register index_reg = as_Register(index);
3340 if (disp == 0) {
3341 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3342 } else {
3343 __ lea(rscratch1, Address(base, disp));
3344 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3345 }
3346 }
3347 %}
3348
3349 // mov encodings
3350
3351 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3352 uint32_t con = (uint32_t)$src$$constant;
3353 Register dst_reg = as_Register($dst$$reg);
3354 if (con == 0) {
3355 __ movw(dst_reg, zr);
3356 } else {
3357 __ movw(dst_reg, con);
3358 }
3359 %}
3360
3361 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3362 Register dst_reg = as_Register($dst$$reg);
3363 uint64_t con = (uint64_t)$src$$constant;
3364 if (con == 0) {
3365 __ mov(dst_reg, zr);
3366 } else {
3367 __ mov(dst_reg, con);
3368 }
3369 %}
3370
3371 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3372 Register dst_reg = as_Register($dst$$reg);
3373 address con = (address)$src$$constant;
3374 if (con == nullptr || con == (address)1) {
3375 ShouldNotReachHere();
3376 } else {
3377 relocInfo::relocType rtype = $src->constant_reloc();
3378 if (rtype == relocInfo::oop_type) {
3379 __ movoop(dst_reg, (jobject)con);
3380 } else if (rtype == relocInfo::metadata_type) {
3381 __ mov_metadata(dst_reg, (Metadata*)con);
3382 } else {
3383 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3384 // load fake address constants using a normal move
3385 if (! __ is_valid_AArch64_address(con) ||
3386 con < (address)(uintptr_t)os::vm_page_size()) {
3387 __ mov(dst_reg, con);
3388 } else {
3389 // no reloc so just use adrp and add
3390 uint64_t offset;
3391 __ adrp(dst_reg, con, offset);
3392 __ add(dst_reg, dst_reg, offset);
3393 }
3394 }
3395 }
3396 %}
3397
3398 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3399 Register dst_reg = as_Register($dst$$reg);
3400 __ mov(dst_reg, zr);
3401 %}
3402
3403 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3404 Register dst_reg = as_Register($dst$$reg);
3405 __ mov(dst_reg, (uint64_t)1);
3406 %}
3407
3408 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3409 Register dst_reg = as_Register($dst$$reg);
3410 address con = (address)$src$$constant;
3411 if (con == nullptr) {
3412 ShouldNotReachHere();
3413 } else {
3414 relocInfo::relocType rtype = $src->constant_reloc();
3415 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3416 __ set_narrow_oop(dst_reg, (jobject)con);
3417 }
3418 %}
3419
3420 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3421 Register dst_reg = as_Register($dst$$reg);
3422 __ mov(dst_reg, zr);
3423 %}
3424
3425 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3426 Register dst_reg = as_Register($dst$$reg);
3427 address con = (address)$src$$constant;
3428 if (con == nullptr) {
3429 ShouldNotReachHere();
3430 } else {
3431 relocInfo::relocType rtype = $src->constant_reloc();
3432 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3433 __ set_narrow_klass(dst_reg, (Klass *)con);
3434 }
3435 %}
3436
3437 // arithmetic encodings
3438
3439 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 Register src_reg = as_Register($src1$$reg);
3442 int32_t con = (int32_t)$src2$$constant;
3443 // add has primary == 0, subtract has primary == 1
3444 if ($primary) { con = -con; }
3445 if (con < 0) {
3446 __ subw(dst_reg, src_reg, -con);
3447 } else {
3448 __ addw(dst_reg, src_reg, con);
3449 }
3450 %}
3451
3452 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 Register src_reg = as_Register($src1$$reg);
3455 int32_t con = (int32_t)$src2$$constant;
3456 // add has primary == 0, subtract has primary == 1
3457 if ($primary) { con = -con; }
3458 if (con < 0) {
3459 __ sub(dst_reg, src_reg, -con);
3460 } else {
3461 __ add(dst_reg, src_reg, con);
3462 }
3463 %}
3464
3465 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3466 Register dst_reg = as_Register($dst$$reg);
3467 Register src1_reg = as_Register($src1$$reg);
3468 Register src2_reg = as_Register($src2$$reg);
3469 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3470 %}
3471
3472 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3473 Register dst_reg = as_Register($dst$$reg);
3474 Register src1_reg = as_Register($src1$$reg);
3475 Register src2_reg = as_Register($src2$$reg);
3476 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3477 %}
3478
3479 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src1_reg = as_Register($src1$$reg);
3482 Register src2_reg = as_Register($src2$$reg);
3483 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3484 %}
3485
3486 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src1_reg = as_Register($src1$$reg);
3489 Register src2_reg = as_Register($src2$$reg);
3490 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3491 %}
3492
3493 // compare instruction encodings
3494
3495 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3496 Register reg1 = as_Register($src1$$reg);
3497 Register reg2 = as_Register($src2$$reg);
3498 __ cmpw(reg1, reg2);
3499 %}
3500
3501 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3502 Register reg = as_Register($src1$$reg);
3503 int32_t val = $src2$$constant;
3504 if (val >= 0) {
3505 __ subsw(zr, reg, val);
3506 } else {
3507 __ addsw(zr, reg, -val);
3508 }
3509 %}
3510
3511 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3512 Register reg1 = as_Register($src1$$reg);
3513 uint32_t val = (uint32_t)$src2$$constant;
3514 __ movw(rscratch1, val);
3515 __ cmpw(reg1, rscratch1);
3516 %}
3517
3518 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3519 Register reg1 = as_Register($src1$$reg);
3520 Register reg2 = as_Register($src2$$reg);
3521 __ cmp(reg1, reg2);
3522 %}
3523
3524 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3525 Register reg = as_Register($src1$$reg);
3526 int64_t val = $src2$$constant;
3527 if (val >= 0) {
3528 __ subs(zr, reg, val);
3529 } else if (val != -val) {
3530 __ adds(zr, reg, -val);
3531 } else {
3532 // aargh, Long.MIN_VALUE is a special case
3533 __ orr(rscratch1, zr, (uint64_t)val);
3534 __ subs(zr, reg, rscratch1);
3535 }
3536 %}
3537
3538 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3539 Register reg1 = as_Register($src1$$reg);
3540 uint64_t val = (uint64_t)$src2$$constant;
3541 __ mov(rscratch1, val);
3542 __ cmp(reg1, rscratch1);
3543 %}
3544
3545 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3546 Register reg1 = as_Register($src1$$reg);
3547 Register reg2 = as_Register($src2$$reg);
3548 __ cmp(reg1, reg2);
3549 %}
3550
3551 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3552 Register reg1 = as_Register($src1$$reg);
3553 Register reg2 = as_Register($src2$$reg);
3554 __ cmpw(reg1, reg2);
3555 %}
3556
3557 enc_class aarch64_enc_testp(iRegP src) %{
3558 Register reg = as_Register($src$$reg);
3559 __ cmp(reg, zr);
3560 %}
3561
3562 enc_class aarch64_enc_testn(iRegN src) %{
3563 Register reg = as_Register($src$$reg);
3564 __ cmpw(reg, zr);
3565 %}
3566
3567 enc_class aarch64_enc_b(label lbl) %{
3568 Label *L = $lbl$$label;
3569 __ b(*L);
3570 %}
3571
3572 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3573 Label *L = $lbl$$label;
3574 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3575 %}
3576
3577 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3578 Label *L = $lbl$$label;
3579 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3580 %}
3581
3582 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3583 %{
3584 Register sub_reg = as_Register($sub$$reg);
3585 Register super_reg = as_Register($super$$reg);
3586 Register temp_reg = as_Register($temp$$reg);
3587 Register result_reg = as_Register($result$$reg);
3588
3589 Label miss;
3590 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3591 nullptr, &miss,
3592 /*set_cond_codes:*/ true);
3593 if ($primary) {
3594 __ mov(result_reg, zr);
3595 }
3596 __ bind(miss);
3597 %}
3598
3599 enc_class aarch64_enc_java_static_call(method meth) %{
3600 address addr = (address)$meth$$method;
3601 address call;
3602 if (!_method) {
3603 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3604 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3605 if (call == nullptr) {
3606 ciEnv::current()->record_failure("CodeCache is full");
3607 return;
3608 }
3609 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3610 // The NOP here is purely to ensure that eliding a call to
3611 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3612 __ nop();
3613 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3614 } else {
3615 int method_index = resolved_method_index(masm);
3616 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3617 : static_call_Relocation::spec(method_index);
3618 call = __ trampoline_call(Address(addr, rspec));
3619 if (call == nullptr) {
3620 ciEnv::current()->record_failure("CodeCache is full");
3621 return;
3622 }
3623 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3624 // Calls of the same statically bound method can share
3625 // a stub to the interpreter.
3626 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3627 } else {
3628 // Emit stub for static call
3629 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3630 if (stub == nullptr) {
3631 ciEnv::current()->record_failure("CodeCache is full");
3632 return;
3633 }
3634 }
3635 }
3636
3637 __ post_call_nop();
3638
3639 // Only non uncommon_trap calls need to reinitialize ptrue.
3640 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3641 __ reinitialize_ptrue();
3642 }
3643 %}
3644
3645 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3646 int method_index = resolved_method_index(masm);
3647 address call = __ ic_call((address)$meth$$method, method_index);
3648 if (call == nullptr) {
3649 ciEnv::current()->record_failure("CodeCache is full");
3650 return;
3651 }
3652 __ post_call_nop();
3653 if (Compile::current()->max_vector_size() > 0) {
3654 __ reinitialize_ptrue();
3655 }
3656 %}
3657
3658 enc_class aarch64_enc_call_epilog() %{
3659 if (VerifyStackAtCalls) {
3660 // Check that stack depth is unchanged: find majik cookie on stack
3661 __ call_Unimplemented();
3662 }
3663 %}
3664
3665 enc_class aarch64_enc_java_to_runtime(method meth) %{
3666 // some calls to generated routines (arraycopy code) are scheduled
3667 // by C2 as runtime calls. if so we can call them using a br (they
3668 // will be in a reachable segment) otherwise we have to use a blr
3669 // which loads the absolute address into a register.
3670 address entry = (address)$meth$$method;
3671 CodeBlob *cb = CodeCache::find_blob(entry);
3672 if (cb) {
3673 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3674 if (call == nullptr) {
3675 ciEnv::current()->record_failure("CodeCache is full");
3676 return;
3677 }
3678 __ post_call_nop();
3679 } else {
3680 Label retaddr;
3681 // Make the anchor frame walkable
3682 __ adr(rscratch2, retaddr);
3683 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3684 __ lea(rscratch1, RuntimeAddress(entry));
3685 __ blr(rscratch1);
3686 __ bind(retaddr);
3687 __ post_call_nop();
3688 }
3689 if (Compile::current()->max_vector_size() > 0) {
3690 __ reinitialize_ptrue();
3691 }
3692 %}
3693
3694 enc_class aarch64_enc_rethrow() %{
3695 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3696 %}
3697
3698 enc_class aarch64_enc_ret() %{
3699 #ifdef ASSERT
3700 if (Compile::current()->max_vector_size() > 0) {
3701 __ verify_ptrue();
3702 }
3703 #endif
3704 __ ret(lr);
3705 %}
3706
3707 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3708 Register target_reg = as_Register($jump_target$$reg);
3709 __ br(target_reg);
3710 %}
3711
3712 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3713 Register target_reg = as_Register($jump_target$$reg);
3714 // exception oop should be in r0
3715 // ret addr has been popped into lr
3716 // callee expects it in r3
3717 __ mov(r3, lr);
3718 __ br(target_reg);
3719 %}
3720
3721 %}
3722
3723 //----------FRAME--------------------------------------------------------------
3724 // Definition of frame structure and management information.
3725 //
3726 // S T A C K L A Y O U T Allocators stack-slot number
3727 // | (to get allocators register number
3728 // G Owned by | | v add OptoReg::stack0())
3729 // r CALLER | |
3730 // o | +--------+ pad to even-align allocators stack-slot
3731 // w V | pad0 | numbers; owned by CALLER
3732 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3733 // h ^ | in | 5
3734 // | | args | 4 Holes in incoming args owned by SELF
3735 // | | | | 3
3736 // | | +--------+
3737 // V | | old out| Empty on Intel, window on Sparc
3738 // | old |preserve| Must be even aligned.
3739 // | SP-+--------+----> Matcher::_old_SP, even aligned
3740 // | | in | 3 area for Intel ret address
3741 // Owned by |preserve| Empty on Sparc.
3742 // SELF +--------+
3743 // | | pad2 | 2 pad to align old SP
3744 // | +--------+ 1
3745 // | | locks | 0
3746 // | +--------+----> OptoReg::stack0(), even aligned
3747 // | | pad1 | 11 pad to align new SP
3748 // | +--------+
3749 // | | | 10
3750 // | | spills | 9 spills
3751 // V | | 8 (pad0 slot for callee)
3752 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3753 // ^ | out | 7
3754 // | | args | 6 Holes in outgoing args owned by CALLEE
3755 // Owned by +--------+
3756 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3757 // | new |preserve| Must be even-aligned.
3758 // | SP-+--------+----> Matcher::_new_SP, even aligned
3759 // | | |
3760 //
3761 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3762 // known from SELF's arguments and the Java calling convention.
3763 // Region 6-7 is determined per call site.
3764 // Note 2: If the calling convention leaves holes in the incoming argument
3765 // area, those holes are owned by SELF. Holes in the outgoing area
3766 // are owned by the CALLEE. Holes should not be necessary in the
3767 // incoming area, as the Java calling convention is completely under
3768 // the control of the AD file. Doubles can be sorted and packed to
3769 // avoid holes. Holes in the outgoing arguments may be necessary for
3770 // varargs C calling conventions.
3771 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3772 // even aligned with pad0 as needed.
3773 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3774 // (the latter is true on Intel but is it false on AArch64?)
3775 // region 6-11 is even aligned; it may be padded out more so that
3776 // the region from SP to FP meets the minimum stack alignment.
3777 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3778 // alignment. Region 11, pad1, may be dynamically extended so that
3779 // SP meets the minimum alignment.
3780
3781 frame %{
3782 // These three registers define part of the calling convention
3783 // between compiled code and the interpreter.
3784
3785 // Inline Cache Register or Method for I2C.
3786 inline_cache_reg(R12);
3787
3788 // Number of stack slots consumed by locking an object
3789 sync_stack_slots(2);
3790
3791 // Compiled code's Frame Pointer
3792 frame_pointer(R31);
3793
3794 // Stack alignment requirement
3795 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3796
3797 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3798 // for calls to C. Supports the var-args backing area for register parms.
3799 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3800
3801 // The after-PROLOG location of the return address. Location of
3802 // return address specifies a type (REG or STACK) and a number
3803 // representing the register number (i.e. - use a register name) or
3804 // stack slot.
3805 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3806 // Otherwise, it is above the locks and verification slot and alignment word
3807 // TODO this may well be correct but need to check why that - 2 is there
3808 // ppc port uses 0 but we definitely need to allow for fixed_slots
3809 // which folds in the space used for monitors
3810 return_addr(STACK - 2 +
3811 align_up((Compile::current()->in_preserve_stack_slots() +
3812 Compile::current()->fixed_slots()),
3813 stack_alignment_in_slots()));
3814
3815 // Location of compiled Java return values. Same as C for now.
3816 return_value
3817 %{
3818 // TODO do we allow ideal_reg == Op_RegN???
3819 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3820 "only return normal values");
3821
3822 static const int lo[Op_RegL + 1] = { // enum name
3823 0, // Op_Node
3824 0, // Op_Set
3825 R0_num, // Op_RegN
3826 R0_num, // Op_RegI
3827 R0_num, // Op_RegP
3828 V0_num, // Op_RegF
3829 V0_num, // Op_RegD
3830 R0_num // Op_RegL
3831 };
3832
3833 static const int hi[Op_RegL + 1] = { // enum name
3834 0, // Op_Node
3835 0, // Op_Set
3836 OptoReg::Bad, // Op_RegN
3837 OptoReg::Bad, // Op_RegI
3838 R0_H_num, // Op_RegP
3839 OptoReg::Bad, // Op_RegF
3840 V0_H_num, // Op_RegD
3841 R0_H_num // Op_RegL
3842 };
3843
3844 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3845 %}
3846 %}
3847
3848 //----------ATTRIBUTES---------------------------------------------------------
3849 //----------Operand Attributes-------------------------------------------------
3850 op_attrib op_cost(1); // Required cost attribute
3851
3852 //----------Instruction Attributes---------------------------------------------
3853 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3854 ins_attrib ins_size(32); // Required size attribute (in bits)
3855 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3856 // a non-matching short branch variant
3857 // of some long branch?
3858 ins_attrib ins_alignment(4); // Required alignment attribute (must
3859 // be a power of 2) specifies the
3860 // alignment that some part of the
3861 // instruction (not necessarily the
3862 // start) requires. If > 1, a
3863 // compute_padding() function must be
3864 // provided for the instruction
3865
3866 // Whether this node is expanded during code emission into a sequence of
3867 // instructions and the first instruction can perform an implicit null check.
3868 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3869
3870 //----------OPERANDS-----------------------------------------------------------
3871 // Operand definitions must precede instruction definitions for correct parsing
3872 // in the ADLC because operands constitute user defined types which are used in
3873 // instruction definitions.
3874
3875 //----------Simple Operands----------------------------------------------------
3876
3877 // Integer operands 32 bit
3878 // 32 bit immediate
3879 operand immI()
3880 %{
3881 match(ConI);
3882
3883 op_cost(0);
3884 format %{ %}
3885 interface(CONST_INTER);
3886 %}
3887
3888 // 32 bit zero
3889 operand immI0()
3890 %{
3891 predicate(n->get_int() == 0);
3892 match(ConI);
3893
3894 op_cost(0);
3895 format %{ %}
3896 interface(CONST_INTER);
3897 %}
3898
3899 // 32 bit unit increment
3900 operand immI_1()
3901 %{
3902 predicate(n->get_int() == 1);
3903 match(ConI);
3904
3905 op_cost(0);
3906 format %{ %}
3907 interface(CONST_INTER);
3908 %}
3909
3910 // 32 bit unit decrement
3911 operand immI_M1()
3912 %{
3913 predicate(n->get_int() == -1);
3914 match(ConI);
3915
3916 op_cost(0);
3917 format %{ %}
3918 interface(CONST_INTER);
3919 %}
3920
3921 // Shift values for add/sub extension shift
3922 operand immIExt()
3923 %{
3924 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3925 match(ConI);
3926
3927 op_cost(0);
3928 format %{ %}
3929 interface(CONST_INTER);
3930 %}
3931
3932 operand immI_gt_1()
3933 %{
3934 predicate(n->get_int() > 1);
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 operand immI_le_4()
3943 %{
3944 predicate(n->get_int() <= 4);
3945 match(ConI);
3946
3947 op_cost(0);
3948 format %{ %}
3949 interface(CONST_INTER);
3950 %}
3951
3952 operand immI_16()
3953 %{
3954 predicate(n->get_int() == 16);
3955 match(ConI);
3956
3957 op_cost(0);
3958 format %{ %}
3959 interface(CONST_INTER);
3960 %}
3961
3962 operand immI_24()
3963 %{
3964 predicate(n->get_int() == 24);
3965 match(ConI);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 operand immI_32()
3973 %{
3974 predicate(n->get_int() == 32);
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_48()
3983 %{
3984 predicate(n->get_int() == 48);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_56()
3993 %{
3994 predicate(n->get_int() == 56);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_255()
4003 %{
4004 predicate(n->get_int() == 255);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_65535()
4013 %{
4014 predicate(n->get_int() == 65535);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_positive()
4023 %{
4024 predicate(n->get_int() > 0);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 // BoolTest condition for signed compare
4033 operand immI_cmp_cond()
4034 %{
4035 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4036 match(ConI);
4037
4038 op_cost(0);
4039 format %{ %}
4040 interface(CONST_INTER);
4041 %}
4042
4043 // BoolTest condition for unsigned compare
4044 operand immI_cmpU_cond()
4045 %{
4046 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4047 match(ConI);
4048
4049 op_cost(0);
4050 format %{ %}
4051 interface(CONST_INTER);
4052 %}
4053
4054 operand immL_255()
4055 %{
4056 predicate(n->get_long() == 255L);
4057 match(ConL);
4058
4059 op_cost(0);
4060 format %{ %}
4061 interface(CONST_INTER);
4062 %}
4063
4064 operand immL_65535()
4065 %{
4066 predicate(n->get_long() == 65535L);
4067 match(ConL);
4068
4069 op_cost(0);
4070 format %{ %}
4071 interface(CONST_INTER);
4072 %}
4073
4074 operand immL_4294967295()
4075 %{
4076 predicate(n->get_long() == 4294967295L);
4077 match(ConL);
4078
4079 op_cost(0);
4080 format %{ %}
4081 interface(CONST_INTER);
4082 %}
4083
4084 operand immL_bitmask()
4085 %{
4086 predicate((n->get_long() != 0)
4087 && ((n->get_long() & 0xc000000000000000l) == 0)
4088 && is_power_of_2(n->get_long() + 1));
4089 match(ConL);
4090
4091 op_cost(0);
4092 format %{ %}
4093 interface(CONST_INTER);
4094 %}
4095
4096 operand immI_bitmask()
4097 %{
4098 predicate((n->get_int() != 0)
4099 && ((n->get_int() & 0xc0000000) == 0)
4100 && is_power_of_2(n->get_int() + 1));
4101 match(ConI);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 operand immL_positive_bitmaskI()
4109 %{
4110 predicate((n->get_long() != 0)
4111 && ((julong)n->get_long() < 0x80000000ULL)
4112 && is_power_of_2(n->get_long() + 1));
4113 match(ConL);
4114
4115 op_cost(0);
4116 format %{ %}
4117 interface(CONST_INTER);
4118 %}
4119
4120 // Scale values for scaled offset addressing modes (up to long but not quad)
4121 operand immIScale()
4122 %{
4123 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4124 match(ConI);
4125
4126 op_cost(0);
4127 format %{ %}
4128 interface(CONST_INTER);
4129 %}
4130
4131 // 5 bit signed integer
4132 operand immI5()
4133 %{
4134 predicate(Assembler::is_simm(n->get_int(), 5));
4135 match(ConI);
4136
4137 op_cost(0);
4138 format %{ %}
4139 interface(CONST_INTER);
4140 %}
4141
4142 // 7 bit unsigned integer
4143 operand immIU7()
4144 %{
4145 predicate(Assembler::is_uimm(n->get_int(), 7));
4146 match(ConI);
4147
4148 op_cost(0);
4149 format %{ %}
4150 interface(CONST_INTER);
4151 %}
4152
4153 // Offset for scaled or unscaled immediate loads and stores
4154 operand immIOffset()
4155 %{
4156 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4157 match(ConI);
4158
4159 op_cost(0);
4160 format %{ %}
4161 interface(CONST_INTER);
4162 %}
4163
4164 operand immIOffset1()
4165 %{
4166 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4167 match(ConI);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 operand immIOffset2()
4175 %{
4176 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 operand immIOffset4()
4185 %{
4186 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immIOffset8()
4195 %{
4196 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4197 match(ConI);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 operand immIOffset16()
4205 %{
4206 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immLOffset()
4215 %{
4216 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4217 match(ConL);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immLoffset1()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4227 match(ConL);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immLoffset2()
4235 %{
4236 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4237 match(ConL);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immLoffset4()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4247 match(ConL);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immLoffset8()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4257 match(ConL);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immLoffset16()
4265 %{
4266 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4267 match(ConL);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 // 5 bit signed long integer
4275 operand immL5()
4276 %{
4277 predicate(Assembler::is_simm(n->get_long(), 5));
4278 match(ConL);
4279
4280 op_cost(0);
4281 format %{ %}
4282 interface(CONST_INTER);
4283 %}
4284
4285 // 7 bit unsigned long integer
4286 operand immLU7()
4287 %{
4288 predicate(Assembler::is_uimm(n->get_long(), 7));
4289 match(ConL);
4290
4291 op_cost(0);
4292 format %{ %}
4293 interface(CONST_INTER);
4294 %}
4295
4296 // 8 bit signed value.
4297 operand immI8()
4298 %{
4299 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4300 match(ConI);
4301
4302 op_cost(0);
4303 format %{ %}
4304 interface(CONST_INTER);
4305 %}
4306
4307 // 8 bit signed value (simm8), or #simm8 LSL 8.
4308 operand immIDupV()
4309 %{
4310 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4311 match(ConI);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 // 8 bit signed value (simm8), or #simm8 LSL 8.
4319 operand immLDupV()
4320 %{
4321 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4322 match(ConL);
4323
4324 op_cost(0);
4325 format %{ %}
4326 interface(CONST_INTER);
4327 %}
4328
4329 // 8 bit signed value (simm8), or #simm8 LSL 8.
4330 operand immHDupV()
4331 %{
4332 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4333 match(ConH);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 // 8 bit integer valid for vector add sub immediate
4341 operand immBAddSubV()
4342 %{
4343 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4344 match(ConI);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 // 32 bit integer valid for add sub immediate
4352 operand immIAddSub()
4353 %{
4354 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4355 match(ConI);
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 // 32 bit integer valid for vector add sub immediate
4362 operand immIAddSubV()
4363 %{
4364 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4365 match(ConI);
4366
4367 op_cost(0);
4368 format %{ %}
4369 interface(CONST_INTER);
4370 %}
4371
4372 // 32 bit unsigned integer valid for logical immediate
4373
4374 operand immBLog()
4375 %{
4376 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4377 match(ConI);
4378
4379 op_cost(0);
4380 format %{ %}
4381 interface(CONST_INTER);
4382 %}
4383
4384 operand immSLog()
4385 %{
4386 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4387 match(ConI);
4388
4389 op_cost(0);
4390 format %{ %}
4391 interface(CONST_INTER);
4392 %}
4393
4394 operand immILog()
4395 %{
4396 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4397 match(ConI);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 // Integer operands 64 bit
4405 // 64 bit immediate
4406 operand immL()
4407 %{
4408 match(ConL);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 // 64 bit zero
4416 operand immL0()
4417 %{
4418 predicate(n->get_long() == 0);
4419 match(ConL);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // 64 bit unit decrement
4427 operand immL_M1()
4428 %{
4429 predicate(n->get_long() == -1);
4430 match(ConL);
4431
4432 op_cost(0);
4433 format %{ %}
4434 interface(CONST_INTER);
4435 %}
4436
4437 // 64 bit integer valid for add sub immediate
4438 operand immLAddSub()
4439 %{
4440 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4441 match(ConL);
4442 op_cost(0);
4443 format %{ %}
4444 interface(CONST_INTER);
4445 %}
4446
4447 // 64 bit integer valid for addv subv immediate
4448 operand immLAddSubV()
4449 %{
4450 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4451 match(ConL);
4452
4453 op_cost(0);
4454 format %{ %}
4455 interface(CONST_INTER);
4456 %}
4457
4458 // 64 bit integer valid for logical immediate
4459 operand immLLog()
4460 %{
4461 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4462 match(ConL);
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 // Long Immediate: low 32-bit mask
4469 operand immL_32bits()
4470 %{
4471 predicate(n->get_long() == 0xFFFFFFFFL);
4472 match(ConL);
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // Pointer operands
4479 // Pointer Immediate
4480 operand immP()
4481 %{
4482 match(ConP);
4483
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 // nullptr Pointer Immediate
4490 operand immP0()
4491 %{
4492 predicate(n->get_ptr() == 0);
4493 match(ConP);
4494
4495 op_cost(0);
4496 format %{ %}
4497 interface(CONST_INTER);
4498 %}
4499
4500 // Pointer Immediate One
4501 // this is used in object initialization (initial object header)
4502 operand immP_1()
4503 %{
4504 predicate(n->get_ptr() == 1);
4505 match(ConP);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // AOT Runtime Constants Address
4513 operand immAOTRuntimeConstantsAddress()
4514 %{
4515 // Check if the address is in the range of AOT Runtime Constants
4516 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4517 match(ConP);
4518
4519 op_cost(0);
4520 format %{ %}
4521 interface(CONST_INTER);
4522 %}
4523
4524 // Float and Double operands
4525 // Double Immediate
4526 operand immD()
4527 %{
4528 match(ConD);
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // Double Immediate: +0.0d
4535 operand immD0()
4536 %{
4537 predicate(jlong_cast(n->getd()) == 0);
4538 match(ConD);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // constant 'double +0.0'.
4546 operand immDPacked()
4547 %{
4548 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4549 match(ConD);
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 // Float Immediate
4556 operand immF()
4557 %{
4558 match(ConF);
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 // Float Immediate: +0.0f.
4565 operand immF0()
4566 %{
4567 predicate(jint_cast(n->getf()) == 0);
4568 match(ConF);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // Half Float (FP16) Immediate
4576 operand immH()
4577 %{
4578 match(ConH);
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 //
4585 operand immFPacked()
4586 %{
4587 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4588 match(ConF);
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 // Narrow pointer operands
4595 // Narrow Pointer Immediate
4596 operand immN()
4597 %{
4598 match(ConN);
4599
4600 op_cost(0);
4601 format %{ %}
4602 interface(CONST_INTER);
4603 %}
4604
4605 // Narrow nullptr Pointer Immediate
4606 operand immN0()
4607 %{
4608 predicate(n->get_narrowcon() == 0);
4609 match(ConN);
4610
4611 op_cost(0);
4612 format %{ %}
4613 interface(CONST_INTER);
4614 %}
4615
4616 operand immNKlass()
4617 %{
4618 match(ConNKlass);
4619
4620 op_cost(0);
4621 format %{ %}
4622 interface(CONST_INTER);
4623 %}
4624
4625 // Integer 32 bit Register Operands
4626 // Integer 32 bitRegister (excludes SP)
4627 operand iRegI()
4628 %{
4629 constraint(ALLOC_IN_RC(any_reg32));
4630 match(RegI);
4631 match(iRegINoSp);
4632 op_cost(0);
4633 format %{ %}
4634 interface(REG_INTER);
4635 %}
4636
4637 // Integer 32 bit Register not Special
4638 operand iRegINoSp()
4639 %{
4640 constraint(ALLOC_IN_RC(no_special_reg32));
4641 match(RegI);
4642 op_cost(0);
4643 format %{ %}
4644 interface(REG_INTER);
4645 %}
4646
4647 // Integer 64 bit Register Operands
4648 // Integer 64 bit Register (includes SP)
4649 operand iRegL()
4650 %{
4651 constraint(ALLOC_IN_RC(any_reg));
4652 match(RegL);
4653 match(iRegLNoSp);
4654 op_cost(0);
4655 format %{ %}
4656 interface(REG_INTER);
4657 %}
4658
4659 // Integer 64 bit Register not Special
4660 operand iRegLNoSp()
4661 %{
4662 constraint(ALLOC_IN_RC(no_special_reg));
4663 match(RegL);
4664 match(iRegL_R0);
4665 format %{ %}
4666 interface(REG_INTER);
4667 %}
4668
4669 // Pointer Register Operands
4670 // Pointer Register
4671 operand iRegP()
4672 %{
4673 constraint(ALLOC_IN_RC(ptr_reg));
4674 match(RegP);
4675 match(iRegPNoSp);
4676 match(iRegP_R0);
4677 //match(iRegP_R2);
4678 //match(iRegP_R4);
4679 match(iRegP_R5);
4680 match(thread_RegP);
4681 op_cost(0);
4682 format %{ %}
4683 interface(REG_INTER);
4684 %}
4685
4686 // Pointer 64 bit Register not Special
4687 operand iRegPNoSp()
4688 %{
4689 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4690 match(RegP);
4691 // match(iRegP);
4692 // match(iRegP_R0);
4693 // match(iRegP_R2);
4694 // match(iRegP_R4);
4695 // match(iRegP_R5);
4696 // match(thread_RegP);
4697 op_cost(0);
4698 format %{ %}
4699 interface(REG_INTER);
4700 %}
4701
4702 // This operand is not allowed to use rfp even if
4703 // rfp is not used to hold the frame pointer.
4704 operand iRegPNoSpNoRfp()
4705 %{
4706 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4707 match(RegP);
4708 match(iRegPNoSp);
4709 op_cost(0);
4710 format %{ %}
4711 interface(REG_INTER);
4712 %}
4713
4714 // Pointer 64 bit Register R0 only
4715 operand iRegP_R0()
4716 %{
4717 constraint(ALLOC_IN_RC(r0_reg));
4718 match(RegP);
4719 // match(iRegP);
4720 match(iRegPNoSp);
4721 op_cost(0);
4722 format %{ %}
4723 interface(REG_INTER);
4724 %}
4725
4726 // Pointer 64 bit Register R1 only
4727 operand iRegP_R1()
4728 %{
4729 constraint(ALLOC_IN_RC(r1_reg));
4730 match(RegP);
4731 // match(iRegP);
4732 match(iRegPNoSp);
4733 op_cost(0);
4734 format %{ %}
4735 interface(REG_INTER);
4736 %}
4737
4738 // Pointer 64 bit Register R2 only
4739 operand iRegP_R2()
4740 %{
4741 constraint(ALLOC_IN_RC(r2_reg));
4742 match(RegP);
4743 // match(iRegP);
4744 match(iRegPNoSp);
4745 op_cost(0);
4746 format %{ %}
4747 interface(REG_INTER);
4748 %}
4749
4750 // Pointer 64 bit Register R3 only
4751 operand iRegP_R3()
4752 %{
4753 constraint(ALLOC_IN_RC(r3_reg));
4754 match(RegP);
4755 // match(iRegP);
4756 match(iRegPNoSp);
4757 op_cost(0);
4758 format %{ %}
4759 interface(REG_INTER);
4760 %}
4761
4762 // Pointer 64 bit Register R4 only
4763 operand iRegP_R4()
4764 %{
4765 constraint(ALLOC_IN_RC(r4_reg));
4766 match(RegP);
4767 // match(iRegP);
4768 match(iRegPNoSp);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // Pointer 64 bit Register R5 only
4775 operand iRegP_R5()
4776 %{
4777 constraint(ALLOC_IN_RC(r5_reg));
4778 match(RegP);
4779 // match(iRegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R10 only
4787 operand iRegP_R10()
4788 %{
4789 constraint(ALLOC_IN_RC(r10_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Long 64 bit Register R0 only
4799 operand iRegL_R0()
4800 %{
4801 constraint(ALLOC_IN_RC(r0_reg));
4802 match(RegL);
4803 match(iRegLNoSp);
4804 op_cost(0);
4805 format %{ %}
4806 interface(REG_INTER);
4807 %}
4808
4809 // Long 64 bit Register R11 only
4810 operand iRegL_R11()
4811 %{
4812 constraint(ALLOC_IN_RC(r11_reg));
4813 match(RegL);
4814 match(iRegLNoSp);
4815 op_cost(0);
4816 format %{ %}
4817 interface(REG_INTER);
4818 %}
4819
4820 // Register R0 only
4821 operand iRegI_R0()
4822 %{
4823 constraint(ALLOC_IN_RC(int_r0_reg));
4824 match(RegI);
4825 match(iRegINoSp);
4826 op_cost(0);
4827 format %{ %}
4828 interface(REG_INTER);
4829 %}
4830
4831 // Register R2 only
4832 operand iRegI_R2()
4833 %{
4834 constraint(ALLOC_IN_RC(int_r2_reg));
4835 match(RegI);
4836 match(iRegINoSp);
4837 op_cost(0);
4838 format %{ %}
4839 interface(REG_INTER);
4840 %}
4841
4842 // Register R3 only
4843 operand iRegI_R3()
4844 %{
4845 constraint(ALLOC_IN_RC(int_r3_reg));
4846 match(RegI);
4847 match(iRegINoSp);
4848 op_cost(0);
4849 format %{ %}
4850 interface(REG_INTER);
4851 %}
4852
4853
4854 // Register R4 only
4855 operand iRegI_R4()
4856 %{
4857 constraint(ALLOC_IN_RC(int_r4_reg));
4858 match(RegI);
4859 match(iRegINoSp);
4860 op_cost(0);
4861 format %{ %}
4862 interface(REG_INTER);
4863 %}
4864
4865
4866 // Pointer Register Operands
4867 // Narrow Pointer Register
4868 operand iRegN()
4869 %{
4870 constraint(ALLOC_IN_RC(any_reg32));
4871 match(RegN);
4872 match(iRegNNoSp);
4873 op_cost(0);
4874 format %{ %}
4875 interface(REG_INTER);
4876 %}
4877
4878 // Integer 64 bit Register not Special
4879 operand iRegNNoSp()
4880 %{
4881 constraint(ALLOC_IN_RC(no_special_reg32));
4882 match(RegN);
4883 op_cost(0);
4884 format %{ %}
4885 interface(REG_INTER);
4886 %}
4887
4888 // Float Register
4889 // Float register operands
4890 operand vRegF()
4891 %{
4892 constraint(ALLOC_IN_RC(float_reg));
4893 match(RegF);
4894
4895 op_cost(0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900 // Double Register
4901 // Double register operands
4902 operand vRegD()
4903 %{
4904 constraint(ALLOC_IN_RC(double_reg));
4905 match(RegD);
4906
4907 op_cost(0);
4908 format %{ %}
4909 interface(REG_INTER);
4910 %}
4911
4912 // Generic vector class. This will be used for
4913 // all vector operands, including NEON and SVE.
4914 operand vReg()
4915 %{
4916 constraint(ALLOC_IN_RC(dynamic));
4917 match(VecA);
4918 match(VecD);
4919 match(VecX);
4920
4921 op_cost(0);
4922 format %{ %}
4923 interface(REG_INTER);
4924 %}
4925
4926 operand vReg_V10()
4927 %{
4928 constraint(ALLOC_IN_RC(v10_veca_reg));
4929 match(vReg);
4930
4931 op_cost(0);
4932 format %{ %}
4933 interface(REG_INTER);
4934 %}
4935
4936 operand vReg_V11()
4937 %{
4938 constraint(ALLOC_IN_RC(v11_veca_reg));
4939 match(vReg);
4940
4941 op_cost(0);
4942 format %{ %}
4943 interface(REG_INTER);
4944 %}
4945
4946 operand vReg_V12()
4947 %{
4948 constraint(ALLOC_IN_RC(v12_veca_reg));
4949 match(vReg);
4950
4951 op_cost(0);
4952 format %{ %}
4953 interface(REG_INTER);
4954 %}
4955
4956 operand vReg_V13()
4957 %{
4958 constraint(ALLOC_IN_RC(v13_veca_reg));
4959 match(vReg);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 operand vReg_V17()
4967 %{
4968 constraint(ALLOC_IN_RC(v17_veca_reg));
4969 match(vReg);
4970
4971 op_cost(0);
4972 format %{ %}
4973 interface(REG_INTER);
4974 %}
4975
4976 operand vReg_V18()
4977 %{
4978 constraint(ALLOC_IN_RC(v18_veca_reg));
4979 match(vReg);
4980
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 operand vReg_V23()
4987 %{
4988 constraint(ALLOC_IN_RC(v23_veca_reg));
4989 match(vReg);
4990
4991 op_cost(0);
4992 format %{ %}
4993 interface(REG_INTER);
4994 %}
4995
4996 operand vReg_V24()
4997 %{
4998 constraint(ALLOC_IN_RC(v24_veca_reg));
4999 match(vReg);
5000
5001 op_cost(0);
5002 format %{ %}
5003 interface(REG_INTER);
5004 %}
5005
5006 operand vecA()
5007 %{
5008 constraint(ALLOC_IN_RC(vectora_reg));
5009 match(VecA);
5010
5011 op_cost(0);
5012 format %{ %}
5013 interface(REG_INTER);
5014 %}
5015
5016 operand vecD()
5017 %{
5018 constraint(ALLOC_IN_RC(vectord_reg));
5019 match(VecD);
5020
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 operand vecX()
5027 %{
5028 constraint(ALLOC_IN_RC(vectorx_reg));
5029 match(VecX);
5030
5031 op_cost(0);
5032 format %{ %}
5033 interface(REG_INTER);
5034 %}
5035
5036 operand vRegD_V0()
5037 %{
5038 constraint(ALLOC_IN_RC(v0_reg));
5039 match(RegD);
5040 op_cost(0);
5041 format %{ %}
5042 interface(REG_INTER);
5043 %}
5044
5045 operand vRegD_V1()
5046 %{
5047 constraint(ALLOC_IN_RC(v1_reg));
5048 match(RegD);
5049 op_cost(0);
5050 format %{ %}
5051 interface(REG_INTER);
5052 %}
5053
5054 operand vRegD_V2()
5055 %{
5056 constraint(ALLOC_IN_RC(v2_reg));
5057 match(RegD);
5058 op_cost(0);
5059 format %{ %}
5060 interface(REG_INTER);
5061 %}
5062
5063 operand vRegD_V3()
5064 %{
5065 constraint(ALLOC_IN_RC(v3_reg));
5066 match(RegD);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand vRegD_V4()
5073 %{
5074 constraint(ALLOC_IN_RC(v4_reg));
5075 match(RegD);
5076 op_cost(0);
5077 format %{ %}
5078 interface(REG_INTER);
5079 %}
5080
5081 operand vRegD_V5()
5082 %{
5083 constraint(ALLOC_IN_RC(v5_reg));
5084 match(RegD);
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vRegD_V6()
5091 %{
5092 constraint(ALLOC_IN_RC(v6_reg));
5093 match(RegD);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 operand vRegD_V7()
5100 %{
5101 constraint(ALLOC_IN_RC(v7_reg));
5102 match(RegD);
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 operand vRegD_V12()
5109 %{
5110 constraint(ALLOC_IN_RC(v12_reg));
5111 match(RegD);
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117 operand vRegD_V13()
5118 %{
5119 constraint(ALLOC_IN_RC(v13_reg));
5120 match(RegD);
5121 op_cost(0);
5122 format %{ %}
5123 interface(REG_INTER);
5124 %}
5125
5126 operand pReg()
5127 %{
5128 constraint(ALLOC_IN_RC(pr_reg));
5129 match(RegVectMask);
5130 match(pRegGov);
5131 op_cost(0);
5132 format %{ %}
5133 interface(REG_INTER);
5134 %}
5135
5136 operand pRegGov()
5137 %{
5138 constraint(ALLOC_IN_RC(gov_pr));
5139 match(RegVectMask);
5140 match(pReg);
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 operand pRegGov_P0()
5147 %{
5148 constraint(ALLOC_IN_RC(p0_reg));
5149 match(RegVectMask);
5150 op_cost(0);
5151 format %{ %}
5152 interface(REG_INTER);
5153 %}
5154
5155 operand pRegGov_P1()
5156 %{
5157 constraint(ALLOC_IN_RC(p1_reg));
5158 match(RegVectMask);
5159 op_cost(0);
5160 format %{ %}
5161 interface(REG_INTER);
5162 %}
5163
5164 // Flags register, used as output of signed compare instructions
5165
5166 // note that on AArch64 we also use this register as the output for
5167 // for floating point compare instructions (CmpF CmpD). this ensures
5168 // that ordered inequality tests use GT, GE, LT or LE none of which
5169 // pass through cases where the result is unordered i.e. one or both
5170 // inputs to the compare is a NaN. this means that the ideal code can
5171 // replace e.g. a GT with an LE and not end up capturing the NaN case
5172 // (where the comparison should always fail). EQ and NE tests are
5173 // always generated in ideal code so that unordered folds into the NE
5174 // case, matching the behaviour of AArch64 NE.
5175 //
5176 // This differs from x86 where the outputs of FP compares use a
5177 // special FP flags registers and where compares based on this
5178 // register are distinguished into ordered inequalities (cmpOpUCF) and
5179 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5180 // to explicitly handle the unordered case in branches. x86 also has
5181 // to include extra CMoveX rules to accept a cmpOpUCF input.
5182
5183 operand rFlagsReg()
5184 %{
5185 constraint(ALLOC_IN_RC(int_flags));
5186 match(RegFlags);
5187
5188 op_cost(0);
5189 format %{ "RFLAGS" %}
5190 interface(REG_INTER);
5191 %}
5192
5193 // Flags register, used as output of unsigned compare instructions
5194 operand rFlagsRegU()
5195 %{
5196 constraint(ALLOC_IN_RC(int_flags));
5197 match(RegFlags);
5198
5199 op_cost(0);
5200 format %{ "RFLAGSU" %}
5201 interface(REG_INTER);
5202 %}
5203
5204 // Special Registers
5205
5206 // Method Register
5207 operand inline_cache_RegP(iRegP reg)
5208 %{
5209 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5210 match(reg);
5211 match(iRegPNoSp);
5212 op_cost(0);
5213 format %{ %}
5214 interface(REG_INTER);
5215 %}
5216
5217 // Thread Register
5218 operand thread_RegP(iRegP reg)
5219 %{
5220 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5221 match(reg);
5222 op_cost(0);
5223 format %{ %}
5224 interface(REG_INTER);
5225 %}
5226
5227 //----------Memory Operands----------------------------------------------------
5228
5229 operand indirect(iRegP reg)
5230 %{
5231 constraint(ALLOC_IN_RC(ptr_reg));
5232 match(reg);
5233 op_cost(0);
5234 format %{ "[$reg]" %}
5235 interface(MEMORY_INTER) %{
5236 base($reg);
5237 index(0xffffffff);
5238 scale(0x0);
5239 disp(0x0);
5240 %}
5241 %}
5242
5243 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5244 %{
5245 constraint(ALLOC_IN_RC(ptr_reg));
5246 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5247 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5248 op_cost(0);
5249 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5250 interface(MEMORY_INTER) %{
5251 base($reg);
5252 index($ireg);
5253 scale($scale);
5254 disp(0x0);
5255 %}
5256 %}
5257
5258 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5259 %{
5260 constraint(ALLOC_IN_RC(ptr_reg));
5261 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5262 match(AddP reg (LShiftL lreg scale));
5263 op_cost(0);
5264 format %{ "$reg, $lreg lsl($scale)" %}
5265 interface(MEMORY_INTER) %{
5266 base($reg);
5267 index($lreg);
5268 scale($scale);
5269 disp(0x0);
5270 %}
5271 %}
5272
5273 operand indIndexI2L(iRegP reg, iRegI ireg)
5274 %{
5275 constraint(ALLOC_IN_RC(ptr_reg));
5276 match(AddP reg (ConvI2L ireg));
5277 op_cost(0);
5278 format %{ "$reg, $ireg, 0, I2L" %}
5279 interface(MEMORY_INTER) %{
5280 base($reg);
5281 index($ireg);
5282 scale(0x0);
5283 disp(0x0);
5284 %}
5285 %}
5286
5287 operand indIndex(iRegP reg, iRegL lreg)
5288 %{
5289 constraint(ALLOC_IN_RC(ptr_reg));
5290 match(AddP reg lreg);
5291 op_cost(0);
5292 format %{ "$reg, $lreg" %}
5293 interface(MEMORY_INTER) %{
5294 base($reg);
5295 index($lreg);
5296 scale(0x0);
5297 disp(0x0);
5298 %}
5299 %}
5300
5301 operand indOffI1(iRegP reg, immIOffset1 off)
5302 %{
5303 constraint(ALLOC_IN_RC(ptr_reg));
5304 match(AddP reg off);
5305 op_cost(0);
5306 format %{ "[$reg, $off]" %}
5307 interface(MEMORY_INTER) %{
5308 base($reg);
5309 index(0xffffffff);
5310 scale(0x0);
5311 disp($off);
5312 %}
5313 %}
5314
5315 operand indOffI2(iRegP reg, immIOffset2 off)
5316 %{
5317 constraint(ALLOC_IN_RC(ptr_reg));
5318 match(AddP reg off);
5319 op_cost(0);
5320 format %{ "[$reg, $off]" %}
5321 interface(MEMORY_INTER) %{
5322 base($reg);
5323 index(0xffffffff);
5324 scale(0x0);
5325 disp($off);
5326 %}
5327 %}
5328
5329 operand indOffI4(iRegP reg, immIOffset4 off)
5330 %{
5331 constraint(ALLOC_IN_RC(ptr_reg));
5332 match(AddP reg off);
5333 op_cost(0);
5334 format %{ "[$reg, $off]" %}
5335 interface(MEMORY_INTER) %{
5336 base($reg);
5337 index(0xffffffff);
5338 scale(0x0);
5339 disp($off);
5340 %}
5341 %}
5342
5343 operand indOffI8(iRegP reg, immIOffset8 off)
5344 %{
5345 constraint(ALLOC_IN_RC(ptr_reg));
5346 match(AddP reg off);
5347 op_cost(0);
5348 format %{ "[$reg, $off]" %}
5349 interface(MEMORY_INTER) %{
5350 base($reg);
5351 index(0xffffffff);
5352 scale(0x0);
5353 disp($off);
5354 %}
5355 %}
5356
5357 operand indOffI16(iRegP reg, immIOffset16 off)
5358 %{
5359 constraint(ALLOC_IN_RC(ptr_reg));
5360 match(AddP reg off);
5361 op_cost(0);
5362 format %{ "[$reg, $off]" %}
5363 interface(MEMORY_INTER) %{
5364 base($reg);
5365 index(0xffffffff);
5366 scale(0x0);
5367 disp($off);
5368 %}
5369 %}
5370
5371 operand indOffL1(iRegP reg, immLoffset1 off)
5372 %{
5373 constraint(ALLOC_IN_RC(ptr_reg));
5374 match(AddP reg off);
5375 op_cost(0);
5376 format %{ "[$reg, $off]" %}
5377 interface(MEMORY_INTER) %{
5378 base($reg);
5379 index(0xffffffff);
5380 scale(0x0);
5381 disp($off);
5382 %}
5383 %}
5384
5385 operand indOffL2(iRegP reg, immLoffset2 off)
5386 %{
5387 constraint(ALLOC_IN_RC(ptr_reg));
5388 match(AddP reg off);
5389 op_cost(0);
5390 format %{ "[$reg, $off]" %}
5391 interface(MEMORY_INTER) %{
5392 base($reg);
5393 index(0xffffffff);
5394 scale(0x0);
5395 disp($off);
5396 %}
5397 %}
5398
5399 operand indOffL4(iRegP reg, immLoffset4 off)
5400 %{
5401 constraint(ALLOC_IN_RC(ptr_reg));
5402 match(AddP reg off);
5403 op_cost(0);
5404 format %{ "[$reg, $off]" %}
5405 interface(MEMORY_INTER) %{
5406 base($reg);
5407 index(0xffffffff);
5408 scale(0x0);
5409 disp($off);
5410 %}
5411 %}
5412
5413 operand indOffL8(iRegP reg, immLoffset8 off)
5414 %{
5415 constraint(ALLOC_IN_RC(ptr_reg));
5416 match(AddP reg off);
5417 op_cost(0);
5418 format %{ "[$reg, $off]" %}
5419 interface(MEMORY_INTER) %{
5420 base($reg);
5421 index(0xffffffff);
5422 scale(0x0);
5423 disp($off);
5424 %}
5425 %}
5426
5427 operand indOffL16(iRegP reg, immLoffset16 off)
5428 %{
5429 constraint(ALLOC_IN_RC(ptr_reg));
5430 match(AddP reg off);
5431 op_cost(0);
5432 format %{ "[$reg, $off]" %}
5433 interface(MEMORY_INTER) %{
5434 base($reg);
5435 index(0xffffffff);
5436 scale(0x0);
5437 disp($off);
5438 %}
5439 %}
5440
5441 operand indirectX2P(iRegL reg)
5442 %{
5443 constraint(ALLOC_IN_RC(ptr_reg));
5444 match(CastX2P reg);
5445 op_cost(0);
5446 format %{ "[$reg]\t# long -> ptr" %}
5447 interface(MEMORY_INTER) %{
5448 base($reg);
5449 index(0xffffffff);
5450 scale(0x0);
5451 disp(0x0);
5452 %}
5453 %}
5454
5455 operand indOffX2P(iRegL reg, immLOffset off)
5456 %{
5457 constraint(ALLOC_IN_RC(ptr_reg));
5458 match(AddP (CastX2P reg) off);
5459 op_cost(0);
5460 format %{ "[$reg, $off]\t# long -> ptr" %}
5461 interface(MEMORY_INTER) %{
5462 base($reg);
5463 index(0xffffffff);
5464 scale(0x0);
5465 disp($off);
5466 %}
5467 %}
5468
5469 operand indirectN(iRegN reg)
5470 %{
5471 predicate(CompressedOops::shift() == 0);
5472 constraint(ALLOC_IN_RC(ptr_reg));
5473 match(DecodeN reg);
5474 op_cost(0);
5475 format %{ "[$reg]\t# narrow" %}
5476 interface(MEMORY_INTER) %{
5477 base($reg);
5478 index(0xffffffff);
5479 scale(0x0);
5480 disp(0x0);
5481 %}
5482 %}
5483
5484 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5485 %{
5486 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5487 constraint(ALLOC_IN_RC(ptr_reg));
5488 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5489 op_cost(0);
5490 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5491 interface(MEMORY_INTER) %{
5492 base($reg);
5493 index($ireg);
5494 scale($scale);
5495 disp(0x0);
5496 %}
5497 %}
5498
5499 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5500 %{
5501 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5502 constraint(ALLOC_IN_RC(ptr_reg));
5503 match(AddP (DecodeN reg) (LShiftL lreg scale));
5504 op_cost(0);
5505 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5506 interface(MEMORY_INTER) %{
5507 base($reg);
5508 index($lreg);
5509 scale($scale);
5510 disp(0x0);
5511 %}
5512 %}
5513
5514 operand indIndexI2LN(iRegN reg, iRegI ireg)
5515 %{
5516 predicate(CompressedOops::shift() == 0);
5517 constraint(ALLOC_IN_RC(ptr_reg));
5518 match(AddP (DecodeN reg) (ConvI2L ireg));
5519 op_cost(0);
5520 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5521 interface(MEMORY_INTER) %{
5522 base($reg);
5523 index($ireg);
5524 scale(0x0);
5525 disp(0x0);
5526 %}
5527 %}
5528
5529 operand indIndexN(iRegN reg, iRegL lreg)
5530 %{
5531 predicate(CompressedOops::shift() == 0);
5532 constraint(ALLOC_IN_RC(ptr_reg));
5533 match(AddP (DecodeN reg) lreg);
5534 op_cost(0);
5535 format %{ "$reg, $lreg\t# narrow" %}
5536 interface(MEMORY_INTER) %{
5537 base($reg);
5538 index($lreg);
5539 scale(0x0);
5540 disp(0x0);
5541 %}
5542 %}
5543
5544 operand indOffIN(iRegN reg, immIOffset off)
5545 %{
5546 predicate(CompressedOops::shift() == 0);
5547 constraint(ALLOC_IN_RC(ptr_reg));
5548 match(AddP (DecodeN reg) off);
5549 op_cost(0);
5550 format %{ "[$reg, $off]\t# narrow" %}
5551 interface(MEMORY_INTER) %{
5552 base($reg);
5553 index(0xffffffff);
5554 scale(0x0);
5555 disp($off);
5556 %}
5557 %}
5558
5559 operand indOffLN(iRegN reg, immLOffset off)
5560 %{
5561 predicate(CompressedOops::shift() == 0);
5562 constraint(ALLOC_IN_RC(ptr_reg));
5563 match(AddP (DecodeN reg) off);
5564 op_cost(0);
5565 format %{ "[$reg, $off]\t# narrow" %}
5566 interface(MEMORY_INTER) %{
5567 base($reg);
5568 index(0xffffffff);
5569 scale(0x0);
5570 disp($off);
5571 %}
5572 %}
5573
5574
5575 //----------Special Memory Operands--------------------------------------------
5576 // Stack Slot Operand - This operand is used for loading and storing temporary
5577 // values on the stack where a match requires a value to
5578 // flow through memory.
5579 operand stackSlotP(sRegP reg)
5580 %{
5581 constraint(ALLOC_IN_RC(stack_slots));
5582 op_cost(100);
5583 // No match rule because this operand is only generated in matching
5584 // match(RegP);
5585 format %{ "[$reg]" %}
5586 interface(MEMORY_INTER) %{
5587 base(0x1e); // RSP
5588 index(0x0); // No Index
5589 scale(0x0); // No Scale
5590 disp($reg); // Stack Offset
5591 %}
5592 %}
5593
5594 operand stackSlotI(sRegI reg)
5595 %{
5596 constraint(ALLOC_IN_RC(stack_slots));
5597 // No match rule because this operand is only generated in matching
5598 // match(RegI);
5599 format %{ "[$reg]" %}
5600 interface(MEMORY_INTER) %{
5601 base(0x1e); // RSP
5602 index(0x0); // No Index
5603 scale(0x0); // No Scale
5604 disp($reg); // Stack Offset
5605 %}
5606 %}
5607
5608 operand stackSlotF(sRegF reg)
5609 %{
5610 constraint(ALLOC_IN_RC(stack_slots));
5611 // No match rule because this operand is only generated in matching
5612 // match(RegF);
5613 format %{ "[$reg]" %}
5614 interface(MEMORY_INTER) %{
5615 base(0x1e); // RSP
5616 index(0x0); // No Index
5617 scale(0x0); // No Scale
5618 disp($reg); // Stack Offset
5619 %}
5620 %}
5621
5622 operand stackSlotD(sRegD reg)
5623 %{
5624 constraint(ALLOC_IN_RC(stack_slots));
5625 // No match rule because this operand is only generated in matching
5626 // match(RegD);
5627 format %{ "[$reg]" %}
5628 interface(MEMORY_INTER) %{
5629 base(0x1e); // RSP
5630 index(0x0); // No Index
5631 scale(0x0); // No Scale
5632 disp($reg); // Stack Offset
5633 %}
5634 %}
5635
5636 operand stackSlotL(sRegL reg)
5637 %{
5638 constraint(ALLOC_IN_RC(stack_slots));
5639 // No match rule because this operand is only generated in matching
5640 // match(RegL);
5641 format %{ "[$reg]" %}
5642 interface(MEMORY_INTER) %{
5643 base(0x1e); // RSP
5644 index(0x0); // No Index
5645 scale(0x0); // No Scale
5646 disp($reg); // Stack Offset
5647 %}
5648 %}
5649
5650 // Operands for expressing Control Flow
5651 // NOTE: Label is a predefined operand which should not be redefined in
5652 // the AD file. It is generically handled within the ADLC.
5653
5654 //----------Conditional Branch Operands----------------------------------------
5655 // Comparison Op - This is the operation of the comparison, and is limited to
5656 // the following set of codes:
5657 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5658 //
5659 // Other attributes of the comparison, such as unsignedness, are specified
5660 // by the comparison instruction that sets a condition code flags register.
5661 // That result is represented by a flags operand whose subtype is appropriate
5662 // to the unsignedness (etc.) of the comparison.
5663 //
5664 // Later, the instruction which matches both the Comparison Op (a Bool) and
5665 // the flags (produced by the Cmp) specifies the coding of the comparison op
5666 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5667
5668 // used for signed integral comparisons and fp comparisons
5669
5670 operand cmpOp()
5671 %{
5672 match(Bool);
5673
5674 format %{ "" %}
5675 interface(COND_INTER) %{
5676 equal(0x0, "eq");
5677 not_equal(0x1, "ne");
5678 less(0xb, "lt");
5679 greater_equal(0xa, "ge");
5680 less_equal(0xd, "le");
5681 greater(0xc, "gt");
5682 overflow(0x6, "vs");
5683 no_overflow(0x7, "vc");
5684 %}
5685 %}
5686
5687 // used for unsigned integral comparisons
5688
5689 operand cmpOpU()
5690 %{
5691 match(Bool);
5692
5693 format %{ "" %}
5694 interface(COND_INTER) %{
5695 equal(0x0, "eq");
5696 not_equal(0x1, "ne");
5697 less(0x3, "lo");
5698 greater_equal(0x2, "hs");
5699 less_equal(0x9, "ls");
5700 greater(0x8, "hi");
5701 overflow(0x6, "vs");
5702 no_overflow(0x7, "vc");
5703 %}
5704 %}
5705
5706 // used for certain integral comparisons which can be
5707 // converted to cbxx or tbxx instructions
5708
5709 operand cmpOpEqNe()
5710 %{
5711 match(Bool);
5712 op_cost(0);
5713 predicate(n->as_Bool()->_test._test == BoolTest::ne
5714 || n->as_Bool()->_test._test == BoolTest::eq);
5715
5716 format %{ "" %}
5717 interface(COND_INTER) %{
5718 equal(0x0, "eq");
5719 not_equal(0x1, "ne");
5720 less(0xb, "lt");
5721 greater_equal(0xa, "ge");
5722 less_equal(0xd, "le");
5723 greater(0xc, "gt");
5724 overflow(0x6, "vs");
5725 no_overflow(0x7, "vc");
5726 %}
5727 %}
5728
5729 // used for certain integral comparisons which can be
5730 // converted to cbxx or tbxx instructions
5731
5732 operand cmpOpLtGe()
5733 %{
5734 match(Bool);
5735 op_cost(0);
5736
5737 predicate(n->as_Bool()->_test._test == BoolTest::lt
5738 || n->as_Bool()->_test._test == BoolTest::ge);
5739
5740 format %{ "" %}
5741 interface(COND_INTER) %{
5742 equal(0x0, "eq");
5743 not_equal(0x1, "ne");
5744 less(0xb, "lt");
5745 greater_equal(0xa, "ge");
5746 less_equal(0xd, "le");
5747 greater(0xc, "gt");
5748 overflow(0x6, "vs");
5749 no_overflow(0x7, "vc");
5750 %}
5751 %}
5752
5753 // used for certain unsigned integral comparisons which can be
5754 // converted to cbxx or tbxx instructions
5755
5756 operand cmpOpUEqNeLeGt()
5757 %{
5758 match(Bool);
5759 op_cost(0);
5760
5761 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5762 n->as_Bool()->_test._test == BoolTest::ne ||
5763 n->as_Bool()->_test._test == BoolTest::le ||
5764 n->as_Bool()->_test._test == BoolTest::gt);
5765
5766 format %{ "" %}
5767 interface(COND_INTER) %{
5768 equal(0x0, "eq");
5769 not_equal(0x1, "ne");
5770 less(0x3, "lo");
5771 greater_equal(0x2, "hs");
5772 less_equal(0x9, "ls");
5773 greater(0x8, "hi");
5774 overflow(0x6, "vs");
5775 no_overflow(0x7, "vc");
5776 %}
5777 %}
5778
5779 // Special operand allowing long args to int ops to be truncated for free
5780
5781 operand iRegL2I(iRegL reg) %{
5782
5783 op_cost(0);
5784
5785 match(ConvL2I reg);
5786
5787 format %{ "l2i($reg)" %}
5788
5789 interface(REG_INTER)
5790 %}
5791
5792 operand iRegL2P(iRegL reg) %{
5793
5794 op_cost(0);
5795
5796 match(CastX2P reg);
5797
5798 format %{ "l2p($reg)" %}
5799
5800 interface(REG_INTER)
5801 %}
5802
5803 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5804 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5805 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5806 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5807
5808 //----------OPERAND CLASSES----------------------------------------------------
5809 // Operand Classes are groups of operands that are used as to simplify
5810 // instruction definitions by not requiring the AD writer to specify
5811 // separate instructions for every form of operand when the
5812 // instruction accepts multiple operand types with the same basic
5813 // encoding and format. The classic case of this is memory operands.
5814
5815 // memory is used to define read/write location for load/store
5816 // instruction defs. we can turn a memory op into an Address
5817
5818 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5819 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5820
5821 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5822 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5823
5824 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5825 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5826
5827 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5828 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5829
5830 // All of the memory operands. For the pipeline description.
5831 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5832 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5833 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5834
5835
5836 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5837 // operations. it allows the src to be either an iRegI or a (ConvL2I
5838 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5839 // can be elided because the 32-bit instruction will just employ the
5840 // lower 32 bits anyway.
5841 //
5842 // n.b. this does not elide all L2I conversions. if the truncated
5843 // value is consumed by more than one operation then the ConvL2I
5844 // cannot be bundled into the consuming nodes so an l2i gets planted
5845 // (actually a movw $dst $src) and the downstream instructions consume
5846 // the result of the l2i as an iRegI input. That's a shame since the
5847 // movw is actually redundant but its not too costly.
5848
5849 opclass iRegIorL2I(iRegI, iRegL2I);
5850 opclass iRegPorL2P(iRegP, iRegL2P);
5851
5852 //----------PIPELINE-----------------------------------------------------------
5853 // Rules which define the behavior of the target architectures pipeline.
5854
5855 // For specific pipelines, eg A53, define the stages of that pipeline
5856 //pipe_desc(ISS, EX1, EX2, WR);
5857 #define ISS S0
5858 #define EX1 S1
5859 #define EX2 S2
5860 #define WR S3
5861
5862 // Integer ALU reg operation
5863 pipeline %{
5864
5865 attributes %{
5866 // ARM instructions are of fixed length
5867 fixed_size_instructions; // Fixed size instructions TODO does
5868 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5869 // ARM instructions come in 32-bit word units
5870 instruction_unit_size = 4; // An instruction is 4 bytes long
5871 instruction_fetch_unit_size = 64; // The processor fetches one line
5872 instruction_fetch_units = 1; // of 64 bytes
5873 %}
5874
5875 // We don't use an actual pipeline model so don't care about resources
5876 // or description. we do use pipeline classes to introduce fixed
5877 // latencies
5878
5879 //----------RESOURCES----------------------------------------------------------
5880 // Resources are the functional units available to the machine
5881
5882 resources( INS0, INS1, INS01 = INS0 | INS1,
5883 ALU0, ALU1, ALU = ALU0 | ALU1,
5884 MAC,
5885 DIV,
5886 BRANCH,
5887 LDST,
5888 NEON_FP);
5889
5890 //----------PIPELINE DESCRIPTION-----------------------------------------------
5891 // Pipeline Description specifies the stages in the machine's pipeline
5892
5893 // Define the pipeline as a generic 6 stage pipeline
5894 pipe_desc(S0, S1, S2, S3, S4, S5);
5895
5896 //----------PIPELINE CLASSES---------------------------------------------------
5897 // Pipeline Classes describe the stages in which input and output are
5898 // referenced by the hardware pipeline.
5899
5900 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5901 %{
5902 single_instruction;
5903 src1 : S1(read);
5904 src2 : S2(read);
5905 dst : S5(write);
5906 INS01 : ISS;
5907 NEON_FP : S5;
5908 %}
5909
5910 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5911 %{
5912 single_instruction;
5913 src1 : S1(read);
5914 src2 : S2(read);
5915 dst : S5(write);
5916 INS01 : ISS;
5917 NEON_FP : S5;
5918 %}
5919
5920 pipe_class fp_uop_s(vRegF dst, vRegF src)
5921 %{
5922 single_instruction;
5923 src : S1(read);
5924 dst : S5(write);
5925 INS01 : ISS;
5926 NEON_FP : S5;
5927 %}
5928
5929 pipe_class fp_uop_d(vRegD dst, vRegD src)
5930 %{
5931 single_instruction;
5932 src : S1(read);
5933 dst : S5(write);
5934 INS01 : ISS;
5935 NEON_FP : S5;
5936 %}
5937
5938 pipe_class fp_d2f(vRegF dst, vRegD src)
5939 %{
5940 single_instruction;
5941 src : S1(read);
5942 dst : S5(write);
5943 INS01 : ISS;
5944 NEON_FP : S5;
5945 %}
5946
5947 pipe_class fp_f2d(vRegD dst, vRegF src)
5948 %{
5949 single_instruction;
5950 src : S1(read);
5951 dst : S5(write);
5952 INS01 : ISS;
5953 NEON_FP : S5;
5954 %}
5955
5956 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5957 %{
5958 single_instruction;
5959 src : S1(read);
5960 dst : S5(write);
5961 INS01 : ISS;
5962 NEON_FP : S5;
5963 %}
5964
5965 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5966 %{
5967 single_instruction;
5968 src : S1(read);
5969 dst : S5(write);
5970 INS01 : ISS;
5971 NEON_FP : S5;
5972 %}
5973
5974 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5975 %{
5976 single_instruction;
5977 src : S1(read);
5978 dst : S5(write);
5979 INS01 : ISS;
5980 NEON_FP : S5;
5981 %}
5982
5983 pipe_class fp_l2f(vRegF dst, iRegL src)
5984 %{
5985 single_instruction;
5986 src : S1(read);
5987 dst : S5(write);
5988 INS01 : ISS;
5989 NEON_FP : S5;
5990 %}
5991
5992 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5993 %{
5994 single_instruction;
5995 src : S1(read);
5996 dst : S5(write);
5997 INS01 : ISS;
5998 NEON_FP : S5;
5999 %}
6000
6001 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6002 %{
6003 single_instruction;
6004 src : S1(read);
6005 dst : S5(write);
6006 INS01 : ISS;
6007 NEON_FP : S5;
6008 %}
6009
6010 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6011 %{
6012 single_instruction;
6013 src : S1(read);
6014 dst : S5(write);
6015 INS01 : ISS;
6016 NEON_FP : S5;
6017 %}
6018
6019 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6020 %{
6021 single_instruction;
6022 src : S1(read);
6023 dst : S5(write);
6024 INS01 : ISS;
6025 NEON_FP : S5;
6026 %}
6027
6028 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6029 %{
6030 single_instruction;
6031 src1 : S1(read);
6032 src2 : S2(read);
6033 dst : S5(write);
6034 INS0 : ISS;
6035 NEON_FP : S5;
6036 %}
6037
6038 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6039 %{
6040 single_instruction;
6041 src1 : S1(read);
6042 src2 : S2(read);
6043 dst : S5(write);
6044 INS0 : ISS;
6045 NEON_FP : S5;
6046 %}
6047
6048 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6049 %{
6050 single_instruction;
6051 cr : S1(read);
6052 src1 : S1(read);
6053 src2 : S1(read);
6054 dst : S3(write);
6055 INS01 : ISS;
6056 NEON_FP : S3;
6057 %}
6058
6059 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6060 %{
6061 single_instruction;
6062 cr : S1(read);
6063 src1 : S1(read);
6064 src2 : S1(read);
6065 dst : S3(write);
6066 INS01 : ISS;
6067 NEON_FP : S3;
6068 %}
6069
6070 pipe_class fp_imm_s(vRegF dst)
6071 %{
6072 single_instruction;
6073 dst : S3(write);
6074 INS01 : ISS;
6075 NEON_FP : S3;
6076 %}
6077
6078 pipe_class fp_imm_d(vRegD dst)
6079 %{
6080 single_instruction;
6081 dst : S3(write);
6082 INS01 : ISS;
6083 NEON_FP : S3;
6084 %}
6085
6086 pipe_class fp_load_constant_s(vRegF dst)
6087 %{
6088 single_instruction;
6089 dst : S4(write);
6090 INS01 : ISS;
6091 NEON_FP : S4;
6092 %}
6093
6094 pipe_class fp_load_constant_d(vRegD dst)
6095 %{
6096 single_instruction;
6097 dst : S4(write);
6098 INS01 : ISS;
6099 NEON_FP : S4;
6100 %}
6101
6102 //------- Integer ALU operations --------------------------
6103
6104 // Integer ALU reg-reg operation
6105 // Operands needed in EX1, result generated in EX2
6106 // Eg. ADD x0, x1, x2
6107 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6108 %{
6109 single_instruction;
6110 dst : EX2(write);
6111 src1 : EX1(read);
6112 src2 : EX1(read);
6113 INS01 : ISS; // Dual issue as instruction 0 or 1
6114 ALU : EX2;
6115 %}
6116
6117 // Integer ALU reg-reg operation with constant shift
6118 // Shifted register must be available in LATE_ISS instead of EX1
6119 // Eg. ADD x0, x1, x2, LSL #2
6120 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6121 %{
6122 single_instruction;
6123 dst : EX2(write);
6124 src1 : EX1(read);
6125 src2 : ISS(read);
6126 INS01 : ISS;
6127 ALU : EX2;
6128 %}
6129
6130 // Integer ALU reg operation with constant shift
6131 // Eg. LSL x0, x1, #shift
6132 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6133 %{
6134 single_instruction;
6135 dst : EX2(write);
6136 src1 : ISS(read);
6137 INS01 : ISS;
6138 ALU : EX2;
6139 %}
6140
6141 // Integer ALU reg-reg operation with variable shift
6142 // Both operands must be available in LATE_ISS instead of EX1
6143 // Result is available in EX1 instead of EX2
6144 // Eg. LSLV x0, x1, x2
6145 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6146 %{
6147 single_instruction;
6148 dst : EX1(write);
6149 src1 : ISS(read);
6150 src2 : ISS(read);
6151 INS01 : ISS;
6152 ALU : EX1;
6153 %}
6154
6155 // Integer ALU reg-reg operation with extract
6156 // As for _vshift above, but result generated in EX2
6157 // Eg. EXTR x0, x1, x2, #N
6158 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6159 %{
6160 single_instruction;
6161 dst : EX2(write);
6162 src1 : ISS(read);
6163 src2 : ISS(read);
6164 INS1 : ISS; // Can only dual issue as Instruction 1
6165 ALU : EX1;
6166 %}
6167
6168 // Integer ALU reg operation
6169 // Eg. NEG x0, x1
6170 pipe_class ialu_reg(iRegI dst, iRegI src)
6171 %{
6172 single_instruction;
6173 dst : EX2(write);
6174 src : EX1(read);
6175 INS01 : ISS;
6176 ALU : EX2;
6177 %}
6178
6179 // Integer ALU reg mmediate operation
6180 // Eg. ADD x0, x1, #N
6181 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6182 %{
6183 single_instruction;
6184 dst : EX2(write);
6185 src1 : EX1(read);
6186 INS01 : ISS;
6187 ALU : EX2;
6188 %}
6189
6190 // Integer ALU immediate operation (no source operands)
6191 // Eg. MOV x0, #N
6192 pipe_class ialu_imm(iRegI dst)
6193 %{
6194 single_instruction;
6195 dst : EX1(write);
6196 INS01 : ISS;
6197 ALU : EX1;
6198 %}
6199
6200 //------- Compare operation -------------------------------
6201
6202 // Compare reg-reg
6203 // Eg. CMP x0, x1
6204 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6205 %{
6206 single_instruction;
6207 // fixed_latency(16);
6208 cr : EX2(write);
6209 op1 : EX1(read);
6210 op2 : EX1(read);
6211 INS01 : ISS;
6212 ALU : EX2;
6213 %}
6214
6215 // Compare reg-reg
6216 // Eg. CMP x0, #N
6217 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6218 %{
6219 single_instruction;
6220 // fixed_latency(16);
6221 cr : EX2(write);
6222 op1 : EX1(read);
6223 INS01 : ISS;
6224 ALU : EX2;
6225 %}
6226
6227 //------- Conditional instructions ------------------------
6228
6229 // Conditional no operands
6230 // Eg. CSINC x0, zr, zr, <cond>
6231 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6232 %{
6233 single_instruction;
6234 cr : EX1(read);
6235 dst : EX2(write);
6236 INS01 : ISS;
6237 ALU : EX2;
6238 %}
6239
6240 // Conditional 2 operand
6241 // EG. CSEL X0, X1, X2, <cond>
6242 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6243 %{
6244 single_instruction;
6245 cr : EX1(read);
6246 src1 : EX1(read);
6247 src2 : EX1(read);
6248 dst : EX2(write);
6249 INS01 : ISS;
6250 ALU : EX2;
6251 %}
6252
6253 // Conditional 2 operand
6254 // EG. CSEL X0, X1, X2, <cond>
6255 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6256 %{
6257 single_instruction;
6258 cr : EX1(read);
6259 src : EX1(read);
6260 dst : EX2(write);
6261 INS01 : ISS;
6262 ALU : EX2;
6263 %}
6264
6265 //------- Multiply pipeline operations --------------------
6266
6267 // Multiply reg-reg
6268 // Eg. MUL w0, w1, w2
6269 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6270 %{
6271 single_instruction;
6272 dst : WR(write);
6273 src1 : ISS(read);
6274 src2 : ISS(read);
6275 INS01 : ISS;
6276 MAC : WR;
6277 %}
6278
6279 // Multiply accumulate
6280 // Eg. MADD w0, w1, w2, w3
6281 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6282 %{
6283 single_instruction;
6284 dst : WR(write);
6285 src1 : ISS(read);
6286 src2 : ISS(read);
6287 src3 : ISS(read);
6288 INS01 : ISS;
6289 MAC : WR;
6290 %}
6291
6292 // Eg. MUL w0, w1, w2
6293 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6294 %{
6295 single_instruction;
6296 fixed_latency(3); // Maximum latency for 64 bit mul
6297 dst : WR(write);
6298 src1 : ISS(read);
6299 src2 : ISS(read);
6300 INS01 : ISS;
6301 MAC : WR;
6302 %}
6303
6304 // Multiply accumulate
6305 // Eg. MADD w0, w1, w2, w3
6306 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6307 %{
6308 single_instruction;
6309 fixed_latency(3); // Maximum latency for 64 bit mul
6310 dst : WR(write);
6311 src1 : ISS(read);
6312 src2 : ISS(read);
6313 src3 : ISS(read);
6314 INS01 : ISS;
6315 MAC : WR;
6316 %}
6317
6318 //------- Divide pipeline operations --------------------
6319
6320 // Eg. SDIV w0, w1, w2
6321 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6322 %{
6323 single_instruction;
6324 fixed_latency(8); // Maximum latency for 32 bit divide
6325 dst : WR(write);
6326 src1 : ISS(read);
6327 src2 : ISS(read);
6328 INS0 : ISS; // Can only dual issue as instruction 0
6329 DIV : WR;
6330 %}
6331
6332 // Eg. SDIV x0, x1, x2
6333 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6334 %{
6335 single_instruction;
6336 fixed_latency(16); // Maximum latency for 64 bit divide
6337 dst : WR(write);
6338 src1 : ISS(read);
6339 src2 : ISS(read);
6340 INS0 : ISS; // Can only dual issue as instruction 0
6341 DIV : WR;
6342 %}
6343
6344 //------- Load pipeline operations ------------------------
6345
6346 // Load - prefetch
6347 // Eg. PFRM <mem>
6348 pipe_class iload_prefetch(memory mem)
6349 %{
6350 single_instruction;
6351 mem : ISS(read);
6352 INS01 : ISS;
6353 LDST : WR;
6354 %}
6355
6356 // Load - reg, mem
6357 // Eg. LDR x0, <mem>
6358 pipe_class iload_reg_mem(iRegI dst, memory mem)
6359 %{
6360 single_instruction;
6361 dst : WR(write);
6362 mem : ISS(read);
6363 INS01 : ISS;
6364 LDST : WR;
6365 %}
6366
6367 // Load - reg, reg
6368 // Eg. LDR x0, [sp, x1]
6369 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6370 %{
6371 single_instruction;
6372 dst : WR(write);
6373 src : ISS(read);
6374 INS01 : ISS;
6375 LDST : WR;
6376 %}
6377
6378 //------- Store pipeline operations -----------------------
6379
6380 // Store - zr, mem
6381 // Eg. STR zr, <mem>
6382 pipe_class istore_mem(memory mem)
6383 %{
6384 single_instruction;
6385 mem : ISS(read);
6386 INS01 : ISS;
6387 LDST : WR;
6388 %}
6389
6390 // Store - reg, mem
6391 // Eg. STR x0, <mem>
6392 pipe_class istore_reg_mem(iRegI src, memory mem)
6393 %{
6394 single_instruction;
6395 mem : ISS(read);
6396 src : EX2(read);
6397 INS01 : ISS;
6398 LDST : WR;
6399 %}
6400
6401 // Store - reg, reg
6402 // Eg. STR x0, [sp, x1]
6403 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6404 %{
6405 single_instruction;
6406 dst : ISS(read);
6407 src : EX2(read);
6408 INS01 : ISS;
6409 LDST : WR;
6410 %}
6411
6412 //------- Store pipeline operations -----------------------
6413
6414 // Branch
6415 pipe_class pipe_branch()
6416 %{
6417 single_instruction;
6418 INS01 : ISS;
6419 BRANCH : EX1;
6420 %}
6421
6422 // Conditional branch
6423 pipe_class pipe_branch_cond(rFlagsReg cr)
6424 %{
6425 single_instruction;
6426 cr : EX1(read);
6427 INS01 : ISS;
6428 BRANCH : EX1;
6429 %}
6430
6431 // Compare & Branch
6432 // EG. CBZ/CBNZ
6433 pipe_class pipe_cmp_branch(iRegI op1)
6434 %{
6435 single_instruction;
6436 op1 : EX1(read);
6437 INS01 : ISS;
6438 BRANCH : EX1;
6439 %}
6440
6441 //------- Synchronisation operations ----------------------
6442
6443 // Any operation requiring serialization.
6444 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6445 pipe_class pipe_serial()
6446 %{
6447 single_instruction;
6448 force_serialization;
6449 fixed_latency(16);
6450 INS01 : ISS(2); // Cannot dual issue with any other instruction
6451 LDST : WR;
6452 %}
6453
6454 // Generic big/slow expanded idiom - also serialized
6455 pipe_class pipe_slow()
6456 %{
6457 instruction_count(10);
6458 multiple_bundles;
6459 force_serialization;
6460 fixed_latency(16);
6461 INS01 : ISS(2); // Cannot dual issue with any other instruction
6462 LDST : WR;
6463 %}
6464
6465 // Empty pipeline class
6466 pipe_class pipe_class_empty()
6467 %{
6468 single_instruction;
6469 fixed_latency(0);
6470 %}
6471
6472 // Default pipeline class.
6473 pipe_class pipe_class_default()
6474 %{
6475 single_instruction;
6476 fixed_latency(2);
6477 %}
6478
6479 // Pipeline class for compares.
6480 pipe_class pipe_class_compare()
6481 %{
6482 single_instruction;
6483 fixed_latency(16);
6484 %}
6485
6486 // Pipeline class for memory operations.
6487 pipe_class pipe_class_memory()
6488 %{
6489 single_instruction;
6490 fixed_latency(16);
6491 %}
6492
6493 // Pipeline class for call.
6494 pipe_class pipe_class_call()
6495 %{
6496 single_instruction;
6497 fixed_latency(100);
6498 %}
6499
6500 // Define the class for the Nop node.
6501 define %{
6502 MachNop = pipe_class_empty;
6503 %}
6504
6505 %}
6506 //----------INSTRUCTIONS-------------------------------------------------------
6507 //
6508 // match -- States which machine-independent subtree may be replaced
6509 // by this instruction.
6510 // ins_cost -- The estimated cost of this instruction is used by instruction
6511 // selection to identify a minimum cost tree of machine
6512 // instructions that matches a tree of machine-independent
6513 // instructions.
6514 // format -- A string providing the disassembly for this instruction.
6515 // The value of an instruction's operand may be inserted
6516 // by referring to it with a '$' prefix.
6517 // opcode -- Three instruction opcodes may be provided. These are referred
6518 // to within an encode class as $primary, $secondary, and $tertiary
6519 // rrspectively. The primary opcode is commonly used to
6520 // indicate the type of machine instruction, while secondary
6521 // and tertiary are often used for prefix options or addressing
6522 // modes.
6523 // ins_encode -- A list of encode classes with parameters. The encode class
6524 // name must have been defined in an 'enc_class' specification
6525 // in the encode section of the architecture description.
6526
6527 // ============================================================================
6528 // Memory (Load/Store) Instructions
6529
6530 // Load Instructions
6531
6532 // Load Byte (8 bit signed)
6533 instruct loadB(iRegINoSp dst, memory1 mem)
6534 %{
6535 match(Set dst (LoadB mem));
6536 predicate(!needs_acquiring_load(n));
6537
6538 ins_cost(4 * INSN_COST);
6539 format %{ "ldrsbw $dst, $mem\t# byte" %}
6540
6541 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6542
6543 ins_pipe(iload_reg_mem);
6544 %}
6545
6546 // Load Byte (8 bit signed) into long
6547 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6548 %{
6549 match(Set dst (ConvI2L (LoadB mem)));
6550 predicate(!needs_acquiring_load(n->in(1)));
6551
6552 ins_cost(4 * INSN_COST);
6553 format %{ "ldrsb $dst, $mem\t# byte" %}
6554
6555 ins_encode(aarch64_enc_ldrsb(dst, mem));
6556
6557 ins_pipe(iload_reg_mem);
6558 %}
6559
6560 // Load Byte (8 bit unsigned)
6561 instruct loadUB(iRegINoSp dst, memory1 mem)
6562 %{
6563 match(Set dst (LoadUB mem));
6564 predicate(!needs_acquiring_load(n));
6565
6566 ins_cost(4 * INSN_COST);
6567 format %{ "ldrbw $dst, $mem\t# byte" %}
6568
6569 ins_encode(aarch64_enc_ldrb(dst, mem));
6570
6571 ins_pipe(iload_reg_mem);
6572 %}
6573
6574 // Load Byte (8 bit unsigned) into long
6575 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6576 %{
6577 match(Set dst (ConvI2L (LoadUB mem)));
6578 predicate(!needs_acquiring_load(n->in(1)));
6579
6580 ins_cost(4 * INSN_COST);
6581 format %{ "ldrb $dst, $mem\t# byte" %}
6582
6583 ins_encode(aarch64_enc_ldrb(dst, mem));
6584
6585 ins_pipe(iload_reg_mem);
6586 %}
6587
6588 // Load Short (16 bit signed)
6589 instruct loadS(iRegINoSp dst, memory2 mem)
6590 %{
6591 match(Set dst (LoadS mem));
6592 predicate(!needs_acquiring_load(n));
6593
6594 ins_cost(4 * INSN_COST);
6595 format %{ "ldrshw $dst, $mem\t# short" %}
6596
6597 ins_encode(aarch64_enc_ldrshw(dst, mem));
6598
6599 ins_pipe(iload_reg_mem);
6600 %}
6601
6602 // Load Short (16 bit signed) into long
6603 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6604 %{
6605 match(Set dst (ConvI2L (LoadS mem)));
6606 predicate(!needs_acquiring_load(n->in(1)));
6607
6608 ins_cost(4 * INSN_COST);
6609 format %{ "ldrsh $dst, $mem\t# short" %}
6610
6611 ins_encode(aarch64_enc_ldrsh(dst, mem));
6612
6613 ins_pipe(iload_reg_mem);
6614 %}
6615
6616 // Load Char (16 bit unsigned)
6617 instruct loadUS(iRegINoSp dst, memory2 mem)
6618 %{
6619 match(Set dst (LoadUS mem));
6620 predicate(!needs_acquiring_load(n));
6621
6622 ins_cost(4 * INSN_COST);
6623 format %{ "ldrh $dst, $mem\t# short" %}
6624
6625 ins_encode(aarch64_enc_ldrh(dst, mem));
6626
6627 ins_pipe(iload_reg_mem);
6628 %}
6629
6630 // Load Short/Char (16 bit unsigned) into long
6631 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6632 %{
6633 match(Set dst (ConvI2L (LoadUS mem)));
6634 predicate(!needs_acquiring_load(n->in(1)));
6635
6636 ins_cost(4 * INSN_COST);
6637 format %{ "ldrh $dst, $mem\t# short" %}
6638
6639 ins_encode(aarch64_enc_ldrh(dst, mem));
6640
6641 ins_pipe(iload_reg_mem);
6642 %}
6643
6644 // Load Integer (32 bit signed)
6645 instruct loadI(iRegINoSp dst, memory4 mem)
6646 %{
6647 match(Set dst (LoadI mem));
6648 predicate(!needs_acquiring_load(n));
6649
6650 ins_cost(4 * INSN_COST);
6651 format %{ "ldrw $dst, $mem\t# int" %}
6652
6653 ins_encode(aarch64_enc_ldrw(dst, mem));
6654
6655 ins_pipe(iload_reg_mem);
6656 %}
6657
6658 // Load Integer (32 bit signed) into long
6659 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6660 %{
6661 match(Set dst (ConvI2L (LoadI mem)));
6662 predicate(!needs_acquiring_load(n->in(1)));
6663
6664 ins_cost(4 * INSN_COST);
6665 format %{ "ldrsw $dst, $mem\t# int" %}
6666
6667 ins_encode(aarch64_enc_ldrsw(dst, mem));
6668
6669 ins_pipe(iload_reg_mem);
6670 %}
6671
6672 // Load Integer (32 bit unsigned) into long
6673 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6674 %{
6675 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6676 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6677
6678 ins_cost(4 * INSN_COST);
6679 format %{ "ldrw $dst, $mem\t# int" %}
6680
6681 ins_encode(aarch64_enc_ldrw(dst, mem));
6682
6683 ins_pipe(iload_reg_mem);
6684 %}
6685
6686 // Load Long (64 bit signed)
6687 instruct loadL(iRegLNoSp dst, memory8 mem)
6688 %{
6689 match(Set dst (LoadL mem));
6690 predicate(!needs_acquiring_load(n));
6691
6692 ins_cost(4 * INSN_COST);
6693 format %{ "ldr $dst, $mem\t# int" %}
6694
6695 ins_encode(aarch64_enc_ldr(dst, mem));
6696
6697 ins_pipe(iload_reg_mem);
6698 %}
6699
6700 // Load Range
6701 instruct loadRange(iRegINoSp dst, memory4 mem)
6702 %{
6703 match(Set dst (LoadRange mem));
6704
6705 ins_cost(4 * INSN_COST);
6706 format %{ "ldrw $dst, $mem\t# range" %}
6707
6708 ins_encode(aarch64_enc_ldrw(dst, mem));
6709
6710 ins_pipe(iload_reg_mem);
6711 %}
6712
6713 // Load Pointer
6714 instruct loadP(iRegPNoSp dst, memory8 mem)
6715 %{
6716 match(Set dst (LoadP mem));
6717 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6718
6719 ins_cost(4 * INSN_COST);
6720 format %{ "ldr $dst, $mem\t# ptr" %}
6721
6722 ins_encode(aarch64_enc_ldr(dst, mem));
6723
6724 ins_pipe(iload_reg_mem);
6725 %}
6726
6727 // Load Compressed Pointer
6728 instruct loadN(iRegNNoSp dst, memory4 mem)
6729 %{
6730 match(Set dst (LoadN mem));
6731 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6732
6733 ins_cost(4 * INSN_COST);
6734 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6735
6736 ins_encode(aarch64_enc_ldrw(dst, mem));
6737
6738 ins_pipe(iload_reg_mem);
6739 %}
6740
6741 // Load Klass Pointer
6742 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6743 %{
6744 match(Set dst (LoadKlass mem));
6745 predicate(!needs_acquiring_load(n));
6746
6747 ins_cost(4 * INSN_COST);
6748 format %{ "ldr $dst, $mem\t# class" %}
6749
6750 ins_encode(aarch64_enc_ldr(dst, mem));
6751
6752 ins_pipe(iload_reg_mem);
6753 %}
6754
6755 // Load Narrow Klass Pointer
6756 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6757 %{
6758 match(Set dst (LoadNKlass mem));
6759 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6760
6761 ins_cost(4 * INSN_COST);
6762 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6763
6764 ins_encode(aarch64_enc_ldrw(dst, mem));
6765
6766 ins_pipe(iload_reg_mem);
6767 %}
6768
6769 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6770 %{
6771 match(Set dst (LoadNKlass mem));
6772 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6773
6774 ins_cost(4 * INSN_COST);
6775 format %{
6776 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6777 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6778 %}
6779 ins_encode %{
6780 // inlined aarch64_enc_ldrw
6781 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6782 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6783 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6784 %}
6785 ins_pipe(iload_reg_mem);
6786 %}
6787
6788 // Load Float
6789 instruct loadF(vRegF dst, memory4 mem)
6790 %{
6791 match(Set dst (LoadF mem));
6792 predicate(!needs_acquiring_load(n));
6793
6794 ins_cost(4 * INSN_COST);
6795 format %{ "ldrs $dst, $mem\t# float" %}
6796
6797 ins_encode( aarch64_enc_ldrs(dst, mem) );
6798
6799 ins_pipe(pipe_class_memory);
6800 %}
6801
6802 // Load Double
6803 instruct loadD(vRegD dst, memory8 mem)
6804 %{
6805 match(Set dst (LoadD mem));
6806 predicate(!needs_acquiring_load(n));
6807
6808 ins_cost(4 * INSN_COST);
6809 format %{ "ldrd $dst, $mem\t# double" %}
6810
6811 ins_encode( aarch64_enc_ldrd(dst, mem) );
6812
6813 ins_pipe(pipe_class_memory);
6814 %}
6815
6816
6817 // Load Int Constant
6818 instruct loadConI(iRegINoSp dst, immI src)
6819 %{
6820 match(Set dst src);
6821
6822 ins_cost(INSN_COST);
6823 format %{ "mov $dst, $src\t# int" %}
6824
6825 ins_encode( aarch64_enc_movw_imm(dst, src) );
6826
6827 ins_pipe(ialu_imm);
6828 %}
6829
6830 // Load Long Constant
6831 instruct loadConL(iRegLNoSp dst, immL src)
6832 %{
6833 match(Set dst src);
6834
6835 ins_cost(INSN_COST);
6836 format %{ "mov $dst, $src\t# long" %}
6837
6838 ins_encode( aarch64_enc_mov_imm(dst, src) );
6839
6840 ins_pipe(ialu_imm);
6841 %}
6842
6843 // Load Pointer Constant
6844
6845 instruct loadConP(iRegPNoSp dst, immP con)
6846 %{
6847 match(Set dst con);
6848
6849 ins_cost(INSN_COST * 4);
6850 format %{
6851 "mov $dst, $con\t# ptr\n\t"
6852 %}
6853
6854 ins_encode(aarch64_enc_mov_p(dst, con));
6855
6856 ins_pipe(ialu_imm);
6857 %}
6858
6859 // Load Null Pointer Constant
6860
6861 instruct loadConP0(iRegPNoSp dst, immP0 con)
6862 %{
6863 match(Set dst con);
6864
6865 ins_cost(INSN_COST);
6866 format %{ "mov $dst, $con\t# nullptr ptr" %}
6867
6868 ins_encode(aarch64_enc_mov_p0(dst, con));
6869
6870 ins_pipe(ialu_imm);
6871 %}
6872
6873 // Load Pointer Constant One
6874
6875 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6876 %{
6877 match(Set dst con);
6878
6879 ins_cost(INSN_COST);
6880 format %{ "mov $dst, $con\t# nullptr ptr" %}
6881
6882 ins_encode(aarch64_enc_mov_p1(dst, con));
6883
6884 ins_pipe(ialu_imm);
6885 %}
6886
6887 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6888 %{
6889 match(Set dst con);
6890
6891 ins_cost(INSN_COST);
6892 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6893
6894 ins_encode %{
6895 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6896 %}
6897
6898 ins_pipe(ialu_imm);
6899 %}
6900
6901 // Load Narrow Pointer Constant
6902
6903 instruct loadConN(iRegNNoSp dst, immN con)
6904 %{
6905 match(Set dst con);
6906
6907 ins_cost(INSN_COST * 4);
6908 format %{ "mov $dst, $con\t# compressed ptr" %}
6909
6910 ins_encode(aarch64_enc_mov_n(dst, con));
6911
6912 ins_pipe(ialu_imm);
6913 %}
6914
6915 // Load Narrow Null Pointer Constant
6916
6917 instruct loadConN0(iRegNNoSp dst, immN0 con)
6918 %{
6919 match(Set dst con);
6920
6921 ins_cost(INSN_COST);
6922 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6923
6924 ins_encode(aarch64_enc_mov_n0(dst, con));
6925
6926 ins_pipe(ialu_imm);
6927 %}
6928
6929 // Load Narrow Klass Constant
6930
6931 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6932 %{
6933 match(Set dst con);
6934
6935 ins_cost(INSN_COST);
6936 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6937
6938 ins_encode(aarch64_enc_mov_nk(dst, con));
6939
6940 ins_pipe(ialu_imm);
6941 %}
6942
6943 // Load Packed Float Constant
6944
6945 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6946 match(Set dst con);
6947 ins_cost(INSN_COST * 4);
6948 format %{ "fmovs $dst, $con"%}
6949 ins_encode %{
6950 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6951 %}
6952
6953 ins_pipe(fp_imm_s);
6954 %}
6955
6956 // Load Float Constant
6957
6958 instruct loadConF(vRegF dst, immF con) %{
6959 match(Set dst con);
6960
6961 ins_cost(INSN_COST * 4);
6962
6963 format %{
6964 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6965 %}
6966
6967 ins_encode %{
6968 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6969 %}
6970
6971 ins_pipe(fp_load_constant_s);
6972 %}
6973
6974 // Load Packed Double Constant
6975
6976 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6977 match(Set dst con);
6978 ins_cost(INSN_COST);
6979 format %{ "fmovd $dst, $con"%}
6980 ins_encode %{
6981 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6982 %}
6983
6984 ins_pipe(fp_imm_d);
6985 %}
6986
6987 // Load Double Constant
6988
6989 instruct loadConD(vRegD dst, immD con) %{
6990 match(Set dst con);
6991
6992 ins_cost(INSN_COST * 5);
6993 format %{
6994 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6995 %}
6996
6997 ins_encode %{
6998 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6999 %}
7000
7001 ins_pipe(fp_load_constant_d);
7002 %}
7003
7004 // Load Half Float Constant
7005 instruct loadConH(vRegF dst, immH con) %{
7006 match(Set dst con);
7007 format %{ "mov rscratch1, $con\n\t"
7008 "fmov $dst, rscratch1"
7009 %}
7010 ins_encode %{
7011 __ movw(rscratch1, (uint32_t)$con$$constant);
7012 __ fmovs($dst$$FloatRegister, rscratch1);
7013 %}
7014 ins_pipe(pipe_class_default);
7015 %}
7016
7017 // Store Instructions
7018
7019 // Store Byte
7020 instruct storeB(iRegIorL2I src, memory1 mem)
7021 %{
7022 match(Set mem (StoreB mem src));
7023 predicate(!needs_releasing_store(n));
7024
7025 ins_cost(INSN_COST);
7026 format %{ "strb $src, $mem\t# byte" %}
7027
7028 ins_encode(aarch64_enc_strb(src, mem));
7029
7030 ins_pipe(istore_reg_mem);
7031 %}
7032
7033
7034 instruct storeimmB0(immI0 zero, memory1 mem)
7035 %{
7036 match(Set mem (StoreB mem zero));
7037 predicate(!needs_releasing_store(n));
7038
7039 ins_cost(INSN_COST);
7040 format %{ "strb rscractch2, $mem\t# byte" %}
7041
7042 ins_encode(aarch64_enc_strb0(mem));
7043
7044 ins_pipe(istore_mem);
7045 %}
7046
7047 // Store Char/Short
7048 instruct storeC(iRegIorL2I src, memory2 mem)
7049 %{
7050 match(Set mem (StoreC mem src));
7051 predicate(!needs_releasing_store(n));
7052
7053 ins_cost(INSN_COST);
7054 format %{ "strh $src, $mem\t# short" %}
7055
7056 ins_encode(aarch64_enc_strh(src, mem));
7057
7058 ins_pipe(istore_reg_mem);
7059 %}
7060
7061 instruct storeimmC0(immI0 zero, memory2 mem)
7062 %{
7063 match(Set mem (StoreC mem zero));
7064 predicate(!needs_releasing_store(n));
7065
7066 ins_cost(INSN_COST);
7067 format %{ "strh zr, $mem\t# short" %}
7068
7069 ins_encode(aarch64_enc_strh0(mem));
7070
7071 ins_pipe(istore_mem);
7072 %}
7073
7074 // Store Integer
7075
7076 instruct storeI(iRegIorL2I src, memory4 mem)
7077 %{
7078 match(Set mem(StoreI mem src));
7079 predicate(!needs_releasing_store(n));
7080
7081 ins_cost(INSN_COST);
7082 format %{ "strw $src, $mem\t# int" %}
7083
7084 ins_encode(aarch64_enc_strw(src, mem));
7085
7086 ins_pipe(istore_reg_mem);
7087 %}
7088
7089 instruct storeimmI0(immI0 zero, memory4 mem)
7090 %{
7091 match(Set mem(StoreI mem zero));
7092 predicate(!needs_releasing_store(n));
7093
7094 ins_cost(INSN_COST);
7095 format %{ "strw zr, $mem\t# int" %}
7096
7097 ins_encode(aarch64_enc_strw0(mem));
7098
7099 ins_pipe(istore_mem);
7100 %}
7101
7102 // Store Long (64 bit signed)
7103 instruct storeL(iRegL src, memory8 mem)
7104 %{
7105 match(Set mem (StoreL mem src));
7106 predicate(!needs_releasing_store(n));
7107
7108 ins_cost(INSN_COST);
7109 format %{ "str $src, $mem\t# int" %}
7110
7111 ins_encode(aarch64_enc_str(src, mem));
7112
7113 ins_pipe(istore_reg_mem);
7114 %}
7115
7116 // Store Long (64 bit signed)
7117 instruct storeimmL0(immL0 zero, memory8 mem)
7118 %{
7119 match(Set mem (StoreL mem zero));
7120 predicate(!needs_releasing_store(n));
7121
7122 ins_cost(INSN_COST);
7123 format %{ "str zr, $mem\t# int" %}
7124
7125 ins_encode(aarch64_enc_str0(mem));
7126
7127 ins_pipe(istore_mem);
7128 %}
7129
7130 // Store Pointer
7131 instruct storeP(iRegP src, memory8 mem)
7132 %{
7133 match(Set mem (StoreP mem src));
7134 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7135
7136 ins_cost(INSN_COST);
7137 format %{ "str $src, $mem\t# ptr" %}
7138
7139 ins_encode(aarch64_enc_str(src, mem));
7140
7141 ins_pipe(istore_reg_mem);
7142 %}
7143
7144 // Store Pointer
7145 instruct storeimmP0(immP0 zero, memory8 mem)
7146 %{
7147 match(Set mem (StoreP mem zero));
7148 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7149
7150 ins_cost(INSN_COST);
7151 format %{ "str zr, $mem\t# ptr" %}
7152
7153 ins_encode(aarch64_enc_str0(mem));
7154
7155 ins_pipe(istore_mem);
7156 %}
7157
7158 // Store Compressed Pointer
7159 instruct storeN(iRegN src, memory4 mem)
7160 %{
7161 match(Set mem (StoreN mem src));
7162 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7163
7164 ins_cost(INSN_COST);
7165 format %{ "strw $src, $mem\t# compressed ptr" %}
7166
7167 ins_encode(aarch64_enc_strw(src, mem));
7168
7169 ins_pipe(istore_reg_mem);
7170 %}
7171
7172 instruct storeImmN0(immN0 zero, memory4 mem)
7173 %{
7174 match(Set mem (StoreN mem zero));
7175 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7176
7177 ins_cost(INSN_COST);
7178 format %{ "strw zr, $mem\t# compressed ptr" %}
7179
7180 ins_encode(aarch64_enc_strw0(mem));
7181
7182 ins_pipe(istore_mem);
7183 %}
7184
7185 // Store Float
7186 instruct storeF(vRegF src, memory4 mem)
7187 %{
7188 match(Set mem (StoreF mem src));
7189 predicate(!needs_releasing_store(n));
7190
7191 ins_cost(INSN_COST);
7192 format %{ "strs $src, $mem\t# float" %}
7193
7194 ins_encode( aarch64_enc_strs(src, mem) );
7195
7196 ins_pipe(pipe_class_memory);
7197 %}
7198
7199 // TODO
7200 // implement storeImmF0 and storeFImmPacked
7201
7202 // Store Double
7203 instruct storeD(vRegD src, memory8 mem)
7204 %{
7205 match(Set mem (StoreD mem src));
7206 predicate(!needs_releasing_store(n));
7207
7208 ins_cost(INSN_COST);
7209 format %{ "strd $src, $mem\t# double" %}
7210
7211 ins_encode( aarch64_enc_strd(src, mem) );
7212
7213 ins_pipe(pipe_class_memory);
7214 %}
7215
7216 // Store Compressed Klass Pointer
7217 instruct storeNKlass(iRegN src, memory4 mem)
7218 %{
7219 predicate(!needs_releasing_store(n));
7220 match(Set mem (StoreNKlass mem src));
7221
7222 ins_cost(INSN_COST);
7223 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7224
7225 ins_encode(aarch64_enc_strw(src, mem));
7226
7227 ins_pipe(istore_reg_mem);
7228 %}
7229
7230 // TODO
7231 // implement storeImmD0 and storeDImmPacked
7232
7233 // prefetch instructions
7234 // Must be safe to execute with invalid address (cannot fault).
7235
7236 instruct prefetchalloc( memory8 mem ) %{
7237 match(PrefetchAllocation mem);
7238
7239 ins_cost(INSN_COST);
7240 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7241
7242 ins_encode( aarch64_enc_prefetchw(mem) );
7243
7244 ins_pipe(iload_prefetch);
7245 %}
7246
7247 // ---------------- volatile loads and stores ----------------
7248
7249 // Load Byte (8 bit signed)
7250 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7251 %{
7252 match(Set dst (LoadB mem));
7253
7254 ins_cost(VOLATILE_REF_COST);
7255 format %{ "ldarsb $dst, $mem\t# byte" %}
7256
7257 ins_encode(aarch64_enc_ldarsb(dst, mem));
7258
7259 ins_pipe(pipe_serial);
7260 %}
7261
7262 // Load Byte (8 bit signed) into long
7263 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7264 %{
7265 match(Set dst (ConvI2L (LoadB mem)));
7266
7267 ins_cost(VOLATILE_REF_COST);
7268 format %{ "ldarsb $dst, $mem\t# byte" %}
7269
7270 ins_encode(aarch64_enc_ldarsb(dst, mem));
7271
7272 ins_pipe(pipe_serial);
7273 %}
7274
7275 // Load Byte (8 bit unsigned)
7276 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7277 %{
7278 match(Set dst (LoadUB mem));
7279
7280 ins_cost(VOLATILE_REF_COST);
7281 format %{ "ldarb $dst, $mem\t# byte" %}
7282
7283 ins_encode(aarch64_enc_ldarb(dst, mem));
7284
7285 ins_pipe(pipe_serial);
7286 %}
7287
7288 // Load Byte (8 bit unsigned) into long
7289 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7290 %{
7291 match(Set dst (ConvI2L (LoadUB mem)));
7292
7293 ins_cost(VOLATILE_REF_COST);
7294 format %{ "ldarb $dst, $mem\t# byte" %}
7295
7296 ins_encode(aarch64_enc_ldarb(dst, mem));
7297
7298 ins_pipe(pipe_serial);
7299 %}
7300
7301 // Load Short (16 bit signed)
7302 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7303 %{
7304 match(Set dst (LoadS mem));
7305
7306 ins_cost(VOLATILE_REF_COST);
7307 format %{ "ldarshw $dst, $mem\t# short" %}
7308
7309 ins_encode(aarch64_enc_ldarshw(dst, mem));
7310
7311 ins_pipe(pipe_serial);
7312 %}
7313
7314 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7315 %{
7316 match(Set dst (LoadUS mem));
7317
7318 ins_cost(VOLATILE_REF_COST);
7319 format %{ "ldarhw $dst, $mem\t# short" %}
7320
7321 ins_encode(aarch64_enc_ldarhw(dst, mem));
7322
7323 ins_pipe(pipe_serial);
7324 %}
7325
7326 // Load Short/Char (16 bit unsigned) into long
7327 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7328 %{
7329 match(Set dst (ConvI2L (LoadUS mem)));
7330
7331 ins_cost(VOLATILE_REF_COST);
7332 format %{ "ldarh $dst, $mem\t# short" %}
7333
7334 ins_encode(aarch64_enc_ldarh(dst, mem));
7335
7336 ins_pipe(pipe_serial);
7337 %}
7338
7339 // Load Short/Char (16 bit signed) into long
7340 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7341 %{
7342 match(Set dst (ConvI2L (LoadS mem)));
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "ldarh $dst, $mem\t# short" %}
7346
7347 ins_encode(aarch64_enc_ldarsh(dst, mem));
7348
7349 ins_pipe(pipe_serial);
7350 %}
7351
7352 // Load Integer (32 bit signed)
7353 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (LoadI mem));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldarw $dst, $mem\t# int" %}
7359
7360 ins_encode(aarch64_enc_ldarw(dst, mem));
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Integer (32 bit unsigned) into long
7366 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7367 %{
7368 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldarw $dst, $mem\t# int" %}
7372
7373 ins_encode(aarch64_enc_ldarw(dst, mem));
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 // Load Long (64 bit signed)
7379 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7380 %{
7381 match(Set dst (LoadL mem));
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "ldar $dst, $mem\t# int" %}
7385
7386 ins_encode(aarch64_enc_ldar(dst, mem));
7387
7388 ins_pipe(pipe_serial);
7389 %}
7390
7391 // Load Pointer
7392 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7393 %{
7394 match(Set dst (LoadP mem));
7395 predicate(n->as_Load()->barrier_data() == 0);
7396
7397 ins_cost(VOLATILE_REF_COST);
7398 format %{ "ldar $dst, $mem\t# ptr" %}
7399
7400 ins_encode(aarch64_enc_ldar(dst, mem));
7401
7402 ins_pipe(pipe_serial);
7403 %}
7404
7405 // Load Compressed Pointer
7406 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7407 %{
7408 match(Set dst (LoadN mem));
7409 predicate(n->as_Load()->barrier_data() == 0);
7410
7411 ins_cost(VOLATILE_REF_COST);
7412 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7413
7414 ins_encode(aarch64_enc_ldarw(dst, mem));
7415
7416 ins_pipe(pipe_serial);
7417 %}
7418
7419 // Load Float
7420 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7421 %{
7422 match(Set dst (LoadF mem));
7423
7424 ins_cost(VOLATILE_REF_COST);
7425 format %{ "ldars $dst, $mem\t# float" %}
7426
7427 ins_encode( aarch64_enc_fldars(dst, mem) );
7428
7429 ins_pipe(pipe_serial);
7430 %}
7431
7432 // Load Double
7433 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7434 %{
7435 match(Set dst (LoadD mem));
7436
7437 ins_cost(VOLATILE_REF_COST);
7438 format %{ "ldard $dst, $mem\t# double" %}
7439
7440 ins_encode( aarch64_enc_fldard(dst, mem) );
7441
7442 ins_pipe(pipe_serial);
7443 %}
7444
7445 // Store Byte
7446 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7447 %{
7448 match(Set mem (StoreB mem src));
7449
7450 ins_cost(VOLATILE_REF_COST);
7451 format %{ "stlrb $src, $mem\t# byte" %}
7452
7453 ins_encode(aarch64_enc_stlrb(src, mem));
7454
7455 ins_pipe(pipe_class_memory);
7456 %}
7457
7458 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7459 %{
7460 match(Set mem (StoreB mem zero));
7461
7462 ins_cost(VOLATILE_REF_COST);
7463 format %{ "stlrb zr, $mem\t# byte" %}
7464
7465 ins_encode(aarch64_enc_stlrb0(mem));
7466
7467 ins_pipe(pipe_class_memory);
7468 %}
7469
7470 // Store Char/Short
7471 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7472 %{
7473 match(Set mem (StoreC mem src));
7474
7475 ins_cost(VOLATILE_REF_COST);
7476 format %{ "stlrh $src, $mem\t# short" %}
7477
7478 ins_encode(aarch64_enc_stlrh(src, mem));
7479
7480 ins_pipe(pipe_class_memory);
7481 %}
7482
7483 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7484 %{
7485 match(Set mem (StoreC mem zero));
7486
7487 ins_cost(VOLATILE_REF_COST);
7488 format %{ "stlrh zr, $mem\t# short" %}
7489
7490 ins_encode(aarch64_enc_stlrh0(mem));
7491
7492 ins_pipe(pipe_class_memory);
7493 %}
7494
7495 // Store Integer
7496
7497 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7498 %{
7499 match(Set mem(StoreI mem src));
7500
7501 ins_cost(VOLATILE_REF_COST);
7502 format %{ "stlrw $src, $mem\t# int" %}
7503
7504 ins_encode(aarch64_enc_stlrw(src, mem));
7505
7506 ins_pipe(pipe_class_memory);
7507 %}
7508
7509 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7510 %{
7511 match(Set mem(StoreI mem zero));
7512
7513 ins_cost(VOLATILE_REF_COST);
7514 format %{ "stlrw zr, $mem\t# int" %}
7515
7516 ins_encode(aarch64_enc_stlrw0(mem));
7517
7518 ins_pipe(pipe_class_memory);
7519 %}
7520
7521 // Store Long (64 bit signed)
7522 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7523 %{
7524 match(Set mem (StoreL mem src));
7525
7526 ins_cost(VOLATILE_REF_COST);
7527 format %{ "stlr $src, $mem\t# int" %}
7528
7529 ins_encode(aarch64_enc_stlr(src, mem));
7530
7531 ins_pipe(pipe_class_memory);
7532 %}
7533
7534 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7535 %{
7536 match(Set mem (StoreL mem zero));
7537
7538 ins_cost(VOLATILE_REF_COST);
7539 format %{ "stlr zr, $mem\t# int" %}
7540
7541 ins_encode(aarch64_enc_stlr0(mem));
7542
7543 ins_pipe(pipe_class_memory);
7544 %}
7545
7546 // Store Pointer
7547 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7548 %{
7549 match(Set mem (StoreP mem src));
7550 predicate(n->as_Store()->barrier_data() == 0);
7551
7552 ins_cost(VOLATILE_REF_COST);
7553 format %{ "stlr $src, $mem\t# ptr" %}
7554
7555 ins_encode(aarch64_enc_stlr(src, mem));
7556
7557 ins_pipe(pipe_class_memory);
7558 %}
7559
7560 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7561 %{
7562 match(Set mem (StoreP mem zero));
7563 predicate(n->as_Store()->barrier_data() == 0);
7564
7565 ins_cost(VOLATILE_REF_COST);
7566 format %{ "stlr zr, $mem\t# ptr" %}
7567
7568 ins_encode(aarch64_enc_stlr0(mem));
7569
7570 ins_pipe(pipe_class_memory);
7571 %}
7572
7573 // Store Compressed Pointer
7574 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7575 %{
7576 match(Set mem (StoreN mem src));
7577 predicate(n->as_Store()->barrier_data() == 0);
7578
7579 ins_cost(VOLATILE_REF_COST);
7580 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7581
7582 ins_encode(aarch64_enc_stlrw(src, mem));
7583
7584 ins_pipe(pipe_class_memory);
7585 %}
7586
7587 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7588 %{
7589 match(Set mem (StoreN mem zero));
7590 predicate(n->as_Store()->barrier_data() == 0);
7591
7592 ins_cost(VOLATILE_REF_COST);
7593 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7594
7595 ins_encode(aarch64_enc_stlrw0(mem));
7596
7597 ins_pipe(pipe_class_memory);
7598 %}
7599
7600 // Store Float
7601 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7602 %{
7603 match(Set mem (StoreF mem src));
7604
7605 ins_cost(VOLATILE_REF_COST);
7606 format %{ "stlrs $src, $mem\t# float" %}
7607
7608 ins_encode( aarch64_enc_fstlrs(src, mem) );
7609
7610 ins_pipe(pipe_class_memory);
7611 %}
7612
7613 // TODO
7614 // implement storeImmF0 and storeFImmPacked
7615
7616 // Store Double
7617 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7618 %{
7619 match(Set mem (StoreD mem src));
7620
7621 ins_cost(VOLATILE_REF_COST);
7622 format %{ "stlrd $src, $mem\t# double" %}
7623
7624 ins_encode( aarch64_enc_fstlrd(src, mem) );
7625
7626 ins_pipe(pipe_class_memory);
7627 %}
7628
7629 // ---------------- end of volatile loads and stores ----------------
7630
7631 instruct cacheWB(indirect addr)
7632 %{
7633 predicate(VM_Version::supports_data_cache_line_flush());
7634 match(CacheWB addr);
7635
7636 ins_cost(100);
7637 format %{"cache wb $addr" %}
7638 ins_encode %{
7639 assert($addr->index_position() < 0, "should be");
7640 assert($addr$$disp == 0, "should be");
7641 __ cache_wb(Address($addr$$base$$Register, 0));
7642 %}
7643 ins_pipe(pipe_slow); // XXX
7644 %}
7645
7646 instruct cacheWBPreSync()
7647 %{
7648 predicate(VM_Version::supports_data_cache_line_flush());
7649 match(CacheWBPreSync);
7650
7651 ins_cost(100);
7652 format %{"cache wb presync" %}
7653 ins_encode %{
7654 __ cache_wbsync(true);
7655 %}
7656 ins_pipe(pipe_slow); // XXX
7657 %}
7658
7659 instruct cacheWBPostSync()
7660 %{
7661 predicate(VM_Version::supports_data_cache_line_flush());
7662 match(CacheWBPostSync);
7663
7664 ins_cost(100);
7665 format %{"cache wb postsync" %}
7666 ins_encode %{
7667 __ cache_wbsync(false);
7668 %}
7669 ins_pipe(pipe_slow); // XXX
7670 %}
7671
7672 // ============================================================================
7673 // BSWAP Instructions
7674
7675 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7676 match(Set dst (ReverseBytesI src));
7677
7678 ins_cost(INSN_COST);
7679 format %{ "revw $dst, $src" %}
7680
7681 ins_encode %{
7682 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7683 %}
7684
7685 ins_pipe(ialu_reg);
7686 %}
7687
7688 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7689 match(Set dst (ReverseBytesL src));
7690
7691 ins_cost(INSN_COST);
7692 format %{ "rev $dst, $src" %}
7693
7694 ins_encode %{
7695 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7696 %}
7697
7698 ins_pipe(ialu_reg);
7699 %}
7700
7701 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7702 match(Set dst (ReverseBytesUS src));
7703
7704 ins_cost(INSN_COST);
7705 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7706
7707 ins_encode %{
7708 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7709 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7710 %}
7711
7712 ins_pipe(ialu_reg);
7713 %}
7714
7715 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7716 match(Set dst (ReverseBytesS src));
7717
7718 ins_cost(INSN_COST);
7719 format %{ "rev16w $dst, $src\n\t"
7720 "sbfmw $dst, $dst, #0, #15" %}
7721
7722 ins_encode %{
7723 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7724 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7725 %}
7726
7727 ins_pipe(ialu_reg);
7728 %}
7729
7730 // ============================================================================
7731 // Zero Count Instructions
7732
7733 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7734 match(Set dst (CountLeadingZerosI src));
7735
7736 ins_cost(INSN_COST);
7737 format %{ "clzw $dst, $src" %}
7738 ins_encode %{
7739 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7740 %}
7741
7742 ins_pipe(ialu_reg);
7743 %}
7744
7745 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7746 match(Set dst (CountLeadingZerosL src));
7747
7748 ins_cost(INSN_COST);
7749 format %{ "clz $dst, $src" %}
7750 ins_encode %{
7751 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7752 %}
7753
7754 ins_pipe(ialu_reg);
7755 %}
7756
7757 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7758 match(Set dst (CountTrailingZerosI src));
7759
7760 ins_cost(INSN_COST * 2);
7761 format %{ "rbitw $dst, $src\n\t"
7762 "clzw $dst, $dst" %}
7763 ins_encode %{
7764 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7765 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7766 %}
7767
7768 ins_pipe(ialu_reg);
7769 %}
7770
7771 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7772 match(Set dst (CountTrailingZerosL src));
7773
7774 ins_cost(INSN_COST * 2);
7775 format %{ "rbit $dst, $src\n\t"
7776 "clz $dst, $dst" %}
7777 ins_encode %{
7778 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7779 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7780 %}
7781
7782 ins_pipe(ialu_reg);
7783 %}
7784
7785 //---------- Population Count Instructions -------------------------------------
7786 //
7787
7788 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7789 match(Set dst (PopCountI src));
7790 effect(TEMP tmp);
7791 ins_cost(INSN_COST * 13);
7792
7793 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7794 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7795 "addv $tmp, $tmp\t# vector (8B)\n\t"
7796 "mov $dst, $tmp\t# vector (1D)" %}
7797 ins_encode %{
7798 __ fmovs($tmp$$FloatRegister, $src$$Register);
7799 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7800 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7801 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7802 %}
7803
7804 ins_pipe(pipe_class_default);
7805 %}
7806
7807 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7808 match(Set dst (PopCountI (LoadI mem)));
7809 effect(TEMP tmp);
7810 ins_cost(INSN_COST * 13);
7811
7812 format %{ "ldrs $tmp, $mem\n\t"
7813 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7814 "addv $tmp, $tmp\t# vector (8B)\n\t"
7815 "mov $dst, $tmp\t# vector (1D)" %}
7816 ins_encode %{
7817 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7818 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7819 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7820 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7821 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7822 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7823 %}
7824
7825 ins_pipe(pipe_class_default);
7826 %}
7827
7828 // Note: Long.bitCount(long) returns an int.
7829 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7830 match(Set dst (PopCountL src));
7831 effect(TEMP tmp);
7832 ins_cost(INSN_COST * 13);
7833
7834 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7835 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7836 "addv $tmp, $tmp\t# vector (8B)\n\t"
7837 "mov $dst, $tmp\t# vector (1D)" %}
7838 ins_encode %{
7839 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7840 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7841 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7842 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7843 %}
7844
7845 ins_pipe(pipe_class_default);
7846 %}
7847
7848 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7849 match(Set dst (PopCountL (LoadL mem)));
7850 effect(TEMP tmp);
7851 ins_cost(INSN_COST * 13);
7852
7853 format %{ "ldrd $tmp, $mem\n\t"
7854 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7855 "addv $tmp, $tmp\t# vector (8B)\n\t"
7856 "mov $dst, $tmp\t# vector (1D)" %}
7857 ins_encode %{
7858 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7859 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7860 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7861 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7862 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7863 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7864 %}
7865
7866 ins_pipe(pipe_class_default);
7867 %}
7868
7869 // ============================================================================
7870 // VerifyVectorAlignment Instruction
7871
7872 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7873 match(Set addr (VerifyVectorAlignment addr mask));
7874 effect(KILL cr);
7875 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7876 ins_encode %{
7877 Label Lskip;
7878 // check if masked bits of addr are zero
7879 __ tst($addr$$Register, $mask$$constant);
7880 __ br(Assembler::EQ, Lskip);
7881 __ stop("verify_vector_alignment found a misaligned vector memory access");
7882 __ bind(Lskip);
7883 %}
7884 ins_pipe(pipe_slow);
7885 %}
7886
7887 // ============================================================================
7888 // MemBar Instruction
7889
7890 instruct load_fence() %{
7891 match(LoadFence);
7892 ins_cost(VOLATILE_REF_COST);
7893
7894 format %{ "load_fence" %}
7895
7896 ins_encode %{
7897 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7898 %}
7899 ins_pipe(pipe_serial);
7900 %}
7901
7902 instruct unnecessary_membar_acquire() %{
7903 predicate(unnecessary_acquire(n));
7904 match(MemBarAcquire);
7905 ins_cost(0);
7906
7907 format %{ "membar_acquire (elided)" %}
7908
7909 ins_encode %{
7910 __ block_comment("membar_acquire (elided)");
7911 %}
7912
7913 ins_pipe(pipe_class_empty);
7914 %}
7915
7916 instruct membar_acquire() %{
7917 match(MemBarAcquire);
7918 ins_cost(VOLATILE_REF_COST);
7919
7920 format %{ "membar_acquire\n\t"
7921 "dmb ishld" %}
7922
7923 ins_encode %{
7924 __ block_comment("membar_acquire");
7925 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7926 %}
7927
7928 ins_pipe(pipe_serial);
7929 %}
7930
7931
7932 instruct membar_acquire_lock() %{
7933 match(MemBarAcquireLock);
7934 ins_cost(VOLATILE_REF_COST);
7935
7936 format %{ "membar_acquire_lock (elided)" %}
7937
7938 ins_encode %{
7939 __ block_comment("membar_acquire_lock (elided)");
7940 %}
7941
7942 ins_pipe(pipe_serial);
7943 %}
7944
7945 instruct store_fence() %{
7946 match(StoreFence);
7947 ins_cost(VOLATILE_REF_COST);
7948
7949 format %{ "store_fence" %}
7950
7951 ins_encode %{
7952 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7953 %}
7954 ins_pipe(pipe_serial);
7955 %}
7956
7957 instruct unnecessary_membar_release() %{
7958 predicate(unnecessary_release(n));
7959 match(MemBarRelease);
7960 ins_cost(0);
7961
7962 format %{ "membar_release (elided)" %}
7963
7964 ins_encode %{
7965 __ block_comment("membar_release (elided)");
7966 %}
7967 ins_pipe(pipe_serial);
7968 %}
7969
7970 instruct membar_release() %{
7971 match(MemBarRelease);
7972 ins_cost(VOLATILE_REF_COST);
7973
7974 format %{ "membar_release\n\t"
7975 "dmb ishst\n\tdmb ishld" %}
7976
7977 ins_encode %{
7978 __ block_comment("membar_release");
7979 // These will be merged if AlwaysMergeDMB is enabled.
7980 __ membar(Assembler::StoreStore);
7981 __ membar(Assembler::LoadStore);
7982 %}
7983 ins_pipe(pipe_serial);
7984 %}
7985
7986 instruct membar_storestore() %{
7987 match(MemBarStoreStore);
7988 match(StoreStoreFence);
7989 ins_cost(VOLATILE_REF_COST);
7990
7991 format %{ "MEMBAR-store-store" %}
7992
7993 ins_encode %{
7994 __ membar(Assembler::StoreStore);
7995 %}
7996 ins_pipe(pipe_serial);
7997 %}
7998
7999 instruct membar_release_lock() %{
8000 match(MemBarReleaseLock);
8001 ins_cost(VOLATILE_REF_COST);
8002
8003 format %{ "membar_release_lock (elided)" %}
8004
8005 ins_encode %{
8006 __ block_comment("membar_release_lock (elided)");
8007 %}
8008
8009 ins_pipe(pipe_serial);
8010 %}
8011
8012 instruct membar_storeload() %{
8013 match(MemBarStoreLoad);
8014 ins_cost(VOLATILE_REF_COST*100);
8015
8016 format %{ "MEMBAR-store-load\n\t"
8017 "dmb ish" %}
8018
8019 ins_encode %{
8020 __ block_comment("membar_storeload");
8021 __ membar(Assembler::StoreLoad);
8022 %}
8023
8024 ins_pipe(pipe_serial);
8025 %}
8026
8027 instruct unnecessary_membar_volatile() %{
8028 predicate(unnecessary_volatile(n));
8029 match(MemBarVolatile);
8030 ins_cost(0);
8031
8032 format %{ "membar_volatile (elided)" %}
8033
8034 ins_encode %{
8035 __ block_comment("membar_volatile (elided)");
8036 %}
8037
8038 ins_pipe(pipe_serial);
8039 %}
8040
8041 instruct membar_volatile() %{
8042 match(MemBarVolatile);
8043 ins_cost(VOLATILE_REF_COST*100);
8044
8045 format %{ "membar_volatile\n\t"
8046 "dmb ish"%}
8047
8048 ins_encode %{
8049 __ block_comment("membar_volatile");
8050 __ membar(Assembler::StoreLoad);
8051 %}
8052
8053 ins_pipe(pipe_serial);
8054 %}
8055
8056 instruct membar_full() %{
8057 match(MemBarFull);
8058 ins_cost(VOLATILE_REF_COST*100);
8059
8060 format %{ "membar_full\n\t"
8061 "dmb ish" %}
8062 ins_encode %{
8063 __ block_comment("membar_full");
8064 __ membar(Assembler::AnyAny);
8065 %}
8066
8067 ins_pipe(pipe_serial);
8068 %}
8069
8070 // ============================================================================
8071 // Cast/Convert Instructions
8072
8073 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8074 match(Set dst (CastX2P src));
8075
8076 ins_cost(INSN_COST);
8077 format %{ "mov $dst, $src\t# long -> ptr" %}
8078
8079 ins_encode %{
8080 if ($dst$$reg != $src$$reg) {
8081 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8082 }
8083 %}
8084
8085 ins_pipe(ialu_reg);
8086 %}
8087
8088 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8089 match(Set dst (CastP2X src));
8090
8091 ins_cost(INSN_COST);
8092 format %{ "mov $dst, $src\t# ptr -> long" %}
8093
8094 ins_encode %{
8095 if ($dst$$reg != $src$$reg) {
8096 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8097 }
8098 %}
8099
8100 ins_pipe(ialu_reg);
8101 %}
8102
8103 // Convert oop into int for vectors alignment masking
8104 instruct convP2I(iRegINoSp dst, iRegP src) %{
8105 match(Set dst (ConvL2I (CastP2X src)));
8106
8107 ins_cost(INSN_COST);
8108 format %{ "movw $dst, $src\t# ptr -> int" %}
8109 ins_encode %{
8110 __ movw($dst$$Register, $src$$Register);
8111 %}
8112
8113 ins_pipe(ialu_reg);
8114 %}
8115
8116 // Convert compressed oop into int for vectors alignment masking
8117 // in case of 32bit oops (heap < 4Gb).
8118 instruct convN2I(iRegINoSp dst, iRegN src)
8119 %{
8120 predicate(CompressedOops::shift() == 0);
8121 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8122
8123 ins_cost(INSN_COST);
8124 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8125 ins_encode %{
8126 __ movw($dst$$Register, $src$$Register);
8127 %}
8128
8129 ins_pipe(ialu_reg);
8130 %}
8131
8132
8133 // Convert oop pointer into compressed form
8134 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8135 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8136 match(Set dst (EncodeP src));
8137 effect(KILL cr);
8138 ins_cost(INSN_COST * 3);
8139 format %{ "encode_heap_oop $dst, $src" %}
8140 ins_encode %{
8141 Register s = $src$$Register;
8142 Register d = $dst$$Register;
8143 __ encode_heap_oop(d, s);
8144 %}
8145 ins_pipe(ialu_reg);
8146 %}
8147
8148 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8149 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8150 match(Set dst (EncodeP src));
8151 ins_cost(INSN_COST * 3);
8152 format %{ "encode_heap_oop_not_null $dst, $src" %}
8153 ins_encode %{
8154 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8155 %}
8156 ins_pipe(ialu_reg);
8157 %}
8158
8159 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8160 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8161 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8162 match(Set dst (DecodeN src));
8163 ins_cost(INSN_COST * 3);
8164 format %{ "decode_heap_oop $dst, $src" %}
8165 ins_encode %{
8166 Register s = $src$$Register;
8167 Register d = $dst$$Register;
8168 __ decode_heap_oop(d, s);
8169 %}
8170 ins_pipe(ialu_reg);
8171 %}
8172
8173 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8174 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8175 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8176 match(Set dst (DecodeN src));
8177 ins_cost(INSN_COST * 3);
8178 format %{ "decode_heap_oop_not_null $dst, $src" %}
8179 ins_encode %{
8180 Register s = $src$$Register;
8181 Register d = $dst$$Register;
8182 __ decode_heap_oop_not_null(d, s);
8183 %}
8184 ins_pipe(ialu_reg);
8185 %}
8186
8187 // n.b. AArch64 implementations of encode_klass_not_null and
8188 // decode_klass_not_null do not modify the flags register so, unlike
8189 // Intel, we don't kill CR as a side effect here
8190
8191 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8192 match(Set dst (EncodePKlass src));
8193
8194 ins_cost(INSN_COST * 3);
8195 format %{ "encode_klass_not_null $dst,$src" %}
8196
8197 ins_encode %{
8198 Register src_reg = as_Register($src$$reg);
8199 Register dst_reg = as_Register($dst$$reg);
8200 __ encode_klass_not_null(dst_reg, src_reg);
8201 %}
8202
8203 ins_pipe(ialu_reg);
8204 %}
8205
8206 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8207 match(Set dst (DecodeNKlass src));
8208
8209 ins_cost(INSN_COST * 3);
8210 format %{ "decode_klass_not_null $dst,$src" %}
8211
8212 ins_encode %{
8213 Register src_reg = as_Register($src$$reg);
8214 Register dst_reg = as_Register($dst$$reg);
8215 if (dst_reg != src_reg) {
8216 __ decode_klass_not_null(dst_reg, src_reg);
8217 } else {
8218 __ decode_klass_not_null(dst_reg);
8219 }
8220 %}
8221
8222 ins_pipe(ialu_reg);
8223 %}
8224
8225 instruct checkCastPP(iRegPNoSp dst)
8226 %{
8227 match(Set dst (CheckCastPP dst));
8228
8229 size(0);
8230 format %{ "# checkcastPP of $dst" %}
8231 ins_encode(/* empty encoding */);
8232 ins_pipe(pipe_class_empty);
8233 %}
8234
8235 instruct castPP(iRegPNoSp dst)
8236 %{
8237 match(Set dst (CastPP dst));
8238
8239 size(0);
8240 format %{ "# castPP of $dst" %}
8241 ins_encode(/* empty encoding */);
8242 ins_pipe(pipe_class_empty);
8243 %}
8244
8245 instruct castII(iRegI dst)
8246 %{
8247 predicate(VerifyConstraintCasts == 0);
8248 match(Set dst (CastII dst));
8249
8250 size(0);
8251 format %{ "# castII of $dst" %}
8252 ins_encode(/* empty encoding */);
8253 ins_cost(0);
8254 ins_pipe(pipe_class_empty);
8255 %}
8256
8257 instruct castII_checked(iRegI dst, rFlagsReg cr)
8258 %{
8259 predicate(VerifyConstraintCasts > 0);
8260 match(Set dst (CastII dst));
8261 effect(KILL cr);
8262
8263 format %{ "# castII_checked of $dst" %}
8264 ins_encode %{
8265 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8266 %}
8267 ins_pipe(pipe_slow);
8268 %}
8269
8270 instruct castLL(iRegL dst)
8271 %{
8272 predicate(VerifyConstraintCasts == 0);
8273 match(Set dst (CastLL dst));
8274
8275 size(0);
8276 format %{ "# castLL of $dst" %}
8277 ins_encode(/* empty encoding */);
8278 ins_cost(0);
8279 ins_pipe(pipe_class_empty);
8280 %}
8281
8282 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8283 %{
8284 predicate(VerifyConstraintCasts > 0);
8285 match(Set dst (CastLL dst));
8286 effect(KILL cr);
8287
8288 format %{ "# castLL_checked of $dst" %}
8289 ins_encode %{
8290 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8291 %}
8292 ins_pipe(pipe_slow);
8293 %}
8294
8295 instruct castHH(vRegF dst)
8296 %{
8297 match(Set dst (CastHH dst));
8298 size(0);
8299 format %{ "# castHH of $dst" %}
8300 ins_encode(/* empty encoding */);
8301 ins_cost(0);
8302 ins_pipe(pipe_class_empty);
8303 %}
8304
8305 instruct castFF(vRegF dst)
8306 %{
8307 match(Set dst (CastFF dst));
8308
8309 size(0);
8310 format %{ "# castFF of $dst" %}
8311 ins_encode(/* empty encoding */);
8312 ins_cost(0);
8313 ins_pipe(pipe_class_empty);
8314 %}
8315
8316 instruct castDD(vRegD dst)
8317 %{
8318 match(Set dst (CastDD dst));
8319
8320 size(0);
8321 format %{ "# castDD of $dst" %}
8322 ins_encode(/* empty encoding */);
8323 ins_cost(0);
8324 ins_pipe(pipe_class_empty);
8325 %}
8326
8327 instruct castVV(vReg dst)
8328 %{
8329 match(Set dst (CastVV dst));
8330
8331 size(0);
8332 format %{ "# castVV of $dst" %}
8333 ins_encode(/* empty encoding */);
8334 ins_cost(0);
8335 ins_pipe(pipe_class_empty);
8336 %}
8337
8338 instruct castVVMask(pRegGov dst)
8339 %{
8340 match(Set dst (CastVV dst));
8341
8342 size(0);
8343 format %{ "# castVV of $dst" %}
8344 ins_encode(/* empty encoding */);
8345 ins_cost(0);
8346 ins_pipe(pipe_class_empty);
8347 %}
8348
8349 // Manifest a CmpU result in an integer register.
8350 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8351 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8352 %{
8353 match(Set dst (CmpU3 src1 src2));
8354 effect(KILL flags);
8355
8356 ins_cost(INSN_COST * 3);
8357 format %{
8358 "cmpw $src1, $src2\n\t"
8359 "csetw $dst, ne\n\t"
8360 "cnegw $dst, lo\t# CmpU3(reg)"
8361 %}
8362 ins_encode %{
8363 __ cmpw($src1$$Register, $src2$$Register);
8364 __ csetw($dst$$Register, Assembler::NE);
8365 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8366 %}
8367
8368 ins_pipe(pipe_class_default);
8369 %}
8370
8371 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8372 %{
8373 match(Set dst (CmpU3 src1 src2));
8374 effect(KILL flags);
8375
8376 ins_cost(INSN_COST * 3);
8377 format %{
8378 "subsw zr, $src1, $src2\n\t"
8379 "csetw $dst, ne\n\t"
8380 "cnegw $dst, lo\t# CmpU3(imm)"
8381 %}
8382 ins_encode %{
8383 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8384 __ csetw($dst$$Register, Assembler::NE);
8385 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8386 %}
8387
8388 ins_pipe(pipe_class_default);
8389 %}
8390
8391 // Manifest a CmpUL result in an integer register.
8392 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8393 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8394 %{
8395 match(Set dst (CmpUL3 src1 src2));
8396 effect(KILL flags);
8397
8398 ins_cost(INSN_COST * 3);
8399 format %{
8400 "cmp $src1, $src2\n\t"
8401 "csetw $dst, ne\n\t"
8402 "cnegw $dst, lo\t# CmpUL3(reg)"
8403 %}
8404 ins_encode %{
8405 __ cmp($src1$$Register, $src2$$Register);
8406 __ csetw($dst$$Register, Assembler::NE);
8407 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8408 %}
8409
8410 ins_pipe(pipe_class_default);
8411 %}
8412
8413 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8414 %{
8415 match(Set dst (CmpUL3 src1 src2));
8416 effect(KILL flags);
8417
8418 ins_cost(INSN_COST * 3);
8419 format %{
8420 "subs zr, $src1, $src2\n\t"
8421 "csetw $dst, ne\n\t"
8422 "cnegw $dst, lo\t# CmpUL3(imm)"
8423 %}
8424 ins_encode %{
8425 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8426 __ csetw($dst$$Register, Assembler::NE);
8427 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8428 %}
8429
8430 ins_pipe(pipe_class_default);
8431 %}
8432
8433 // Manifest a CmpL result in an integer register.
8434 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8435 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8436 %{
8437 match(Set dst (CmpL3 src1 src2));
8438 effect(KILL flags);
8439
8440 ins_cost(INSN_COST * 3);
8441 format %{
8442 "cmp $src1, $src2\n\t"
8443 "csetw $dst, ne\n\t"
8444 "cnegw $dst, lt\t# CmpL3(reg)"
8445 %}
8446 ins_encode %{
8447 __ cmp($src1$$Register, $src2$$Register);
8448 __ csetw($dst$$Register, Assembler::NE);
8449 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8450 %}
8451
8452 ins_pipe(pipe_class_default);
8453 %}
8454
8455 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8456 %{
8457 match(Set dst (CmpL3 src1 src2));
8458 effect(KILL flags);
8459
8460 ins_cost(INSN_COST * 3);
8461 format %{
8462 "subs zr, $src1, $src2\n\t"
8463 "csetw $dst, ne\n\t"
8464 "cnegw $dst, lt\t# CmpL3(imm)"
8465 %}
8466 ins_encode %{
8467 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8468 __ csetw($dst$$Register, Assembler::NE);
8469 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8470 %}
8471
8472 ins_pipe(pipe_class_default);
8473 %}
8474
8475 // ============================================================================
8476 // Conditional Move Instructions
8477
8478 // n.b. we have identical rules for both a signed compare op (cmpOp)
8479 // and an unsigned compare op (cmpOpU). it would be nice if we could
8480 // define an op class which merged both inputs and use it to type the
8481 // argument to a single rule. unfortunatelyt his fails because the
8482 // opclass does not live up to the COND_INTER interface of its
8483 // component operands. When the generic code tries to negate the
8484 // operand it ends up running the generci Machoper::negate method
8485 // which throws a ShouldNotHappen. So, we have to provide two flavours
8486 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8487
8488 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8489 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8490
8491 ins_cost(INSN_COST * 2);
8492 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8493
8494 ins_encode %{
8495 __ cselw(as_Register($dst$$reg),
8496 as_Register($src2$$reg),
8497 as_Register($src1$$reg),
8498 (Assembler::Condition)$cmp$$cmpcode);
8499 %}
8500
8501 ins_pipe(icond_reg_reg);
8502 %}
8503
8504 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8505 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8506
8507 ins_cost(INSN_COST * 2);
8508 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8509
8510 ins_encode %{
8511 __ cselw(as_Register($dst$$reg),
8512 as_Register($src2$$reg),
8513 as_Register($src1$$reg),
8514 (Assembler::Condition)$cmp$$cmpcode);
8515 %}
8516
8517 ins_pipe(icond_reg_reg);
8518 %}
8519
8520 // special cases where one arg is zero
8521
8522 // n.b. this is selected in preference to the rule above because it
8523 // avoids loading constant 0 into a source register
8524
8525 // TODO
8526 // we ought only to be able to cull one of these variants as the ideal
8527 // transforms ought always to order the zero consistently (to left/right?)
8528
8529 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8530 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8531
8532 ins_cost(INSN_COST * 2);
8533 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8534
8535 ins_encode %{
8536 __ cselw(as_Register($dst$$reg),
8537 as_Register($src$$reg),
8538 zr,
8539 (Assembler::Condition)$cmp$$cmpcode);
8540 %}
8541
8542 ins_pipe(icond_reg);
8543 %}
8544
8545 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8546 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8547
8548 ins_cost(INSN_COST * 2);
8549 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8550
8551 ins_encode %{
8552 __ cselw(as_Register($dst$$reg),
8553 as_Register($src$$reg),
8554 zr,
8555 (Assembler::Condition)$cmp$$cmpcode);
8556 %}
8557
8558 ins_pipe(icond_reg);
8559 %}
8560
8561 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8562 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8563
8564 ins_cost(INSN_COST * 2);
8565 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8566
8567 ins_encode %{
8568 __ cselw(as_Register($dst$$reg),
8569 zr,
8570 as_Register($src$$reg),
8571 (Assembler::Condition)$cmp$$cmpcode);
8572 %}
8573
8574 ins_pipe(icond_reg);
8575 %}
8576
8577 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8578 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8579
8580 ins_cost(INSN_COST * 2);
8581 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8582
8583 ins_encode %{
8584 __ cselw(as_Register($dst$$reg),
8585 zr,
8586 as_Register($src$$reg),
8587 (Assembler::Condition)$cmp$$cmpcode);
8588 %}
8589
8590 ins_pipe(icond_reg);
8591 %}
8592
8593 // special case for creating a boolean 0 or 1
8594
8595 // n.b. this is selected in preference to the rule above because it
8596 // avoids loading constants 0 and 1 into a source register
8597
8598 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8599 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8600
8601 ins_cost(INSN_COST * 2);
8602 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8603
8604 ins_encode %{
8605 // equivalently
8606 // cset(as_Register($dst$$reg),
8607 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8608 __ csincw(as_Register($dst$$reg),
8609 zr,
8610 zr,
8611 (Assembler::Condition)$cmp$$cmpcode);
8612 %}
8613
8614 ins_pipe(icond_none);
8615 %}
8616
8617 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8618 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8619
8620 ins_cost(INSN_COST * 2);
8621 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8622
8623 ins_encode %{
8624 // equivalently
8625 // cset(as_Register($dst$$reg),
8626 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8627 __ csincw(as_Register($dst$$reg),
8628 zr,
8629 zr,
8630 (Assembler::Condition)$cmp$$cmpcode);
8631 %}
8632
8633 ins_pipe(icond_none);
8634 %}
8635
8636 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8637 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8638
8639 ins_cost(INSN_COST * 2);
8640 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8641
8642 ins_encode %{
8643 __ csel(as_Register($dst$$reg),
8644 as_Register($src2$$reg),
8645 as_Register($src1$$reg),
8646 (Assembler::Condition)$cmp$$cmpcode);
8647 %}
8648
8649 ins_pipe(icond_reg_reg);
8650 %}
8651
8652 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8653 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8654
8655 ins_cost(INSN_COST * 2);
8656 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8657
8658 ins_encode %{
8659 __ csel(as_Register($dst$$reg),
8660 as_Register($src2$$reg),
8661 as_Register($src1$$reg),
8662 (Assembler::Condition)$cmp$$cmpcode);
8663 %}
8664
8665 ins_pipe(icond_reg_reg);
8666 %}
8667
8668 // special cases where one arg is zero
8669
8670 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8671 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8672
8673 ins_cost(INSN_COST * 2);
8674 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8675
8676 ins_encode %{
8677 __ csel(as_Register($dst$$reg),
8678 zr,
8679 as_Register($src$$reg),
8680 (Assembler::Condition)$cmp$$cmpcode);
8681 %}
8682
8683 ins_pipe(icond_reg);
8684 %}
8685
8686 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8687 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8688
8689 ins_cost(INSN_COST * 2);
8690 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8691
8692 ins_encode %{
8693 __ csel(as_Register($dst$$reg),
8694 zr,
8695 as_Register($src$$reg),
8696 (Assembler::Condition)$cmp$$cmpcode);
8697 %}
8698
8699 ins_pipe(icond_reg);
8700 %}
8701
8702 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8703 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8704
8705 ins_cost(INSN_COST * 2);
8706 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8707
8708 ins_encode %{
8709 __ csel(as_Register($dst$$reg),
8710 as_Register($src$$reg),
8711 zr,
8712 (Assembler::Condition)$cmp$$cmpcode);
8713 %}
8714
8715 ins_pipe(icond_reg);
8716 %}
8717
8718 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8719 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8720
8721 ins_cost(INSN_COST * 2);
8722 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8723
8724 ins_encode %{
8725 __ csel(as_Register($dst$$reg),
8726 as_Register($src$$reg),
8727 zr,
8728 (Assembler::Condition)$cmp$$cmpcode);
8729 %}
8730
8731 ins_pipe(icond_reg);
8732 %}
8733
8734 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8735 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8736
8737 ins_cost(INSN_COST * 2);
8738 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8739
8740 ins_encode %{
8741 __ csel(as_Register($dst$$reg),
8742 as_Register($src2$$reg),
8743 as_Register($src1$$reg),
8744 (Assembler::Condition)$cmp$$cmpcode);
8745 %}
8746
8747 ins_pipe(icond_reg_reg);
8748 %}
8749
8750 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8751 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8752
8753 ins_cost(INSN_COST * 2);
8754 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8755
8756 ins_encode %{
8757 __ csel(as_Register($dst$$reg),
8758 as_Register($src2$$reg),
8759 as_Register($src1$$reg),
8760 (Assembler::Condition)$cmp$$cmpcode);
8761 %}
8762
8763 ins_pipe(icond_reg_reg);
8764 %}
8765
8766 // special cases where one arg is zero
8767
8768 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8769 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8770
8771 ins_cost(INSN_COST * 2);
8772 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8773
8774 ins_encode %{
8775 __ csel(as_Register($dst$$reg),
8776 zr,
8777 as_Register($src$$reg),
8778 (Assembler::Condition)$cmp$$cmpcode);
8779 %}
8780
8781 ins_pipe(icond_reg);
8782 %}
8783
8784 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8785 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8786
8787 ins_cost(INSN_COST * 2);
8788 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8789
8790 ins_encode %{
8791 __ csel(as_Register($dst$$reg),
8792 zr,
8793 as_Register($src$$reg),
8794 (Assembler::Condition)$cmp$$cmpcode);
8795 %}
8796
8797 ins_pipe(icond_reg);
8798 %}
8799
8800 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8801 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8802
8803 ins_cost(INSN_COST * 2);
8804 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8805
8806 ins_encode %{
8807 __ csel(as_Register($dst$$reg),
8808 as_Register($src$$reg),
8809 zr,
8810 (Assembler::Condition)$cmp$$cmpcode);
8811 %}
8812
8813 ins_pipe(icond_reg);
8814 %}
8815
8816 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8817 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8818
8819 ins_cost(INSN_COST * 2);
8820 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8821
8822 ins_encode %{
8823 __ csel(as_Register($dst$$reg),
8824 as_Register($src$$reg),
8825 zr,
8826 (Assembler::Condition)$cmp$$cmpcode);
8827 %}
8828
8829 ins_pipe(icond_reg);
8830 %}
8831
8832 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8833 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8834
8835 ins_cost(INSN_COST * 2);
8836 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8837
8838 ins_encode %{
8839 __ cselw(as_Register($dst$$reg),
8840 as_Register($src2$$reg),
8841 as_Register($src1$$reg),
8842 (Assembler::Condition)$cmp$$cmpcode);
8843 %}
8844
8845 ins_pipe(icond_reg_reg);
8846 %}
8847
8848 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8849 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8850
8851 ins_cost(INSN_COST * 2);
8852 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8853
8854 ins_encode %{
8855 __ cselw(as_Register($dst$$reg),
8856 as_Register($src2$$reg),
8857 as_Register($src1$$reg),
8858 (Assembler::Condition)$cmp$$cmpcode);
8859 %}
8860
8861 ins_pipe(icond_reg_reg);
8862 %}
8863
8864 // special cases where one arg is zero
8865
8866 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8867 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8868
8869 ins_cost(INSN_COST * 2);
8870 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8871
8872 ins_encode %{
8873 __ cselw(as_Register($dst$$reg),
8874 zr,
8875 as_Register($src$$reg),
8876 (Assembler::Condition)$cmp$$cmpcode);
8877 %}
8878
8879 ins_pipe(icond_reg);
8880 %}
8881
8882 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8883 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8884
8885 ins_cost(INSN_COST * 2);
8886 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8887
8888 ins_encode %{
8889 __ cselw(as_Register($dst$$reg),
8890 zr,
8891 as_Register($src$$reg),
8892 (Assembler::Condition)$cmp$$cmpcode);
8893 %}
8894
8895 ins_pipe(icond_reg);
8896 %}
8897
8898 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8899 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8900
8901 ins_cost(INSN_COST * 2);
8902 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8903
8904 ins_encode %{
8905 __ cselw(as_Register($dst$$reg),
8906 as_Register($src$$reg),
8907 zr,
8908 (Assembler::Condition)$cmp$$cmpcode);
8909 %}
8910
8911 ins_pipe(icond_reg);
8912 %}
8913
8914 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8915 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8916
8917 ins_cost(INSN_COST * 2);
8918 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8919
8920 ins_encode %{
8921 __ cselw(as_Register($dst$$reg),
8922 as_Register($src$$reg),
8923 zr,
8924 (Assembler::Condition)$cmp$$cmpcode);
8925 %}
8926
8927 ins_pipe(icond_reg);
8928 %}
8929
8930 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8931 %{
8932 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8933
8934 ins_cost(INSN_COST * 3);
8935
8936 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8937 ins_encode %{
8938 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8939 __ fcsels(as_FloatRegister($dst$$reg),
8940 as_FloatRegister($src2$$reg),
8941 as_FloatRegister($src1$$reg),
8942 cond);
8943 %}
8944
8945 ins_pipe(fp_cond_reg_reg_s);
8946 %}
8947
8948 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8949 %{
8950 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8951
8952 ins_cost(INSN_COST * 3);
8953
8954 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8955 ins_encode %{
8956 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8957 __ fcsels(as_FloatRegister($dst$$reg),
8958 as_FloatRegister($src2$$reg),
8959 as_FloatRegister($src1$$reg),
8960 cond);
8961 %}
8962
8963 ins_pipe(fp_cond_reg_reg_s);
8964 %}
8965
8966 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8967 %{
8968 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8969
8970 ins_cost(INSN_COST * 3);
8971
8972 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8973 ins_encode %{
8974 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8975 __ fcseld(as_FloatRegister($dst$$reg),
8976 as_FloatRegister($src2$$reg),
8977 as_FloatRegister($src1$$reg),
8978 cond);
8979 %}
8980
8981 ins_pipe(fp_cond_reg_reg_d);
8982 %}
8983
8984 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8985 %{
8986 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8987
8988 ins_cost(INSN_COST * 3);
8989
8990 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8991 ins_encode %{
8992 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8993 __ fcseld(as_FloatRegister($dst$$reg),
8994 as_FloatRegister($src2$$reg),
8995 as_FloatRegister($src1$$reg),
8996 cond);
8997 %}
8998
8999 ins_pipe(fp_cond_reg_reg_d);
9000 %}
9001
9002 // ============================================================================
9003 // Arithmetic Instructions
9004 //
9005
9006 // Integer Addition
9007
9008 // TODO
9009 // these currently employ operations which do not set CR and hence are
9010 // not flagged as killing CR but we would like to isolate the cases
9011 // where we want to set flags from those where we don't. need to work
9012 // out how to do that.
9013
9014 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9015 match(Set dst (AddI src1 src2));
9016
9017 ins_cost(INSN_COST);
9018 format %{ "addw $dst, $src1, $src2" %}
9019
9020 ins_encode %{
9021 __ addw(as_Register($dst$$reg),
9022 as_Register($src1$$reg),
9023 as_Register($src2$$reg));
9024 %}
9025
9026 ins_pipe(ialu_reg_reg);
9027 %}
9028
9029 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9030 match(Set dst (AddI src1 src2));
9031
9032 ins_cost(INSN_COST);
9033 format %{ "addw $dst, $src1, $src2" %}
9034
9035 // use opcode to indicate that this is an add not a sub
9036 opcode(0x0);
9037
9038 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9039
9040 ins_pipe(ialu_reg_imm);
9041 %}
9042
9043 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9044 match(Set dst (AddI (ConvL2I src1) src2));
9045
9046 ins_cost(INSN_COST);
9047 format %{ "addw $dst, $src1, $src2" %}
9048
9049 // use opcode to indicate that this is an add not a sub
9050 opcode(0x0);
9051
9052 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9053
9054 ins_pipe(ialu_reg_imm);
9055 %}
9056
9057 // Pointer Addition
9058 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9059 match(Set dst (AddP src1 src2));
9060
9061 ins_cost(INSN_COST);
9062 format %{ "add $dst, $src1, $src2\t# ptr" %}
9063
9064 ins_encode %{
9065 __ add(as_Register($dst$$reg),
9066 as_Register($src1$$reg),
9067 as_Register($src2$$reg));
9068 %}
9069
9070 ins_pipe(ialu_reg_reg);
9071 %}
9072
9073 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9074 match(Set dst (AddP src1 (ConvI2L src2)));
9075
9076 ins_cost(1.9 * INSN_COST);
9077 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9078
9079 ins_encode %{
9080 __ add(as_Register($dst$$reg),
9081 as_Register($src1$$reg),
9082 as_Register($src2$$reg), ext::sxtw);
9083 %}
9084
9085 ins_pipe(ialu_reg_reg);
9086 %}
9087
9088 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9089 match(Set dst (AddP src1 (LShiftL src2 scale)));
9090
9091 ins_cost(1.9 * INSN_COST);
9092 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9093
9094 ins_encode %{
9095 __ lea(as_Register($dst$$reg),
9096 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9097 Address::lsl($scale$$constant)));
9098 %}
9099
9100 ins_pipe(ialu_reg_reg_shift);
9101 %}
9102
9103 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9104 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9105
9106 ins_cost(1.9 * INSN_COST);
9107 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9108
9109 ins_encode %{
9110 __ lea(as_Register($dst$$reg),
9111 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9112 Address::sxtw($scale$$constant)));
9113 %}
9114
9115 ins_pipe(ialu_reg_reg_shift);
9116 %}
9117
9118 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9119 match(Set dst (LShiftL (ConvI2L src) scale));
9120
9121 ins_cost(INSN_COST);
9122 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9123
9124 ins_encode %{
9125 __ sbfiz(as_Register($dst$$reg),
9126 as_Register($src$$reg),
9127 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9128 %}
9129
9130 ins_pipe(ialu_reg_shift);
9131 %}
9132
9133 // Pointer Immediate Addition
9134 // n.b. this needs to be more expensive than using an indirect memory
9135 // operand
9136 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9137 match(Set dst (AddP src1 src2));
9138
9139 ins_cost(INSN_COST);
9140 format %{ "add $dst, $src1, $src2\t# ptr" %}
9141
9142 // use opcode to indicate that this is an add not a sub
9143 opcode(0x0);
9144
9145 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9146
9147 ins_pipe(ialu_reg_imm);
9148 %}
9149
9150 // Long Addition
9151 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9152
9153 match(Set dst (AddL src1 src2));
9154
9155 ins_cost(INSN_COST);
9156 format %{ "add $dst, $src1, $src2" %}
9157
9158 ins_encode %{
9159 __ add(as_Register($dst$$reg),
9160 as_Register($src1$$reg),
9161 as_Register($src2$$reg));
9162 %}
9163
9164 ins_pipe(ialu_reg_reg);
9165 %}
9166
9167 // No constant pool entries requiredLong Immediate Addition.
9168 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9169 match(Set dst (AddL src1 src2));
9170
9171 ins_cost(INSN_COST);
9172 format %{ "add $dst, $src1, $src2" %}
9173
9174 // use opcode to indicate that this is an add not a sub
9175 opcode(0x0);
9176
9177 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9178
9179 ins_pipe(ialu_reg_imm);
9180 %}
9181
9182 // Integer Subtraction
9183 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9184 match(Set dst (SubI src1 src2));
9185
9186 ins_cost(INSN_COST);
9187 format %{ "subw $dst, $src1, $src2" %}
9188
9189 ins_encode %{
9190 __ subw(as_Register($dst$$reg),
9191 as_Register($src1$$reg),
9192 as_Register($src2$$reg));
9193 %}
9194
9195 ins_pipe(ialu_reg_reg);
9196 %}
9197
9198 // Immediate Subtraction
9199 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9200 match(Set dst (SubI src1 src2));
9201
9202 ins_cost(INSN_COST);
9203 format %{ "subw $dst, $src1, $src2" %}
9204
9205 // use opcode to indicate that this is a sub not an add
9206 opcode(0x1);
9207
9208 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9209
9210 ins_pipe(ialu_reg_imm);
9211 %}
9212
9213 // Long Subtraction
9214 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9215
9216 match(Set dst (SubL src1 src2));
9217
9218 ins_cost(INSN_COST);
9219 format %{ "sub $dst, $src1, $src2" %}
9220
9221 ins_encode %{
9222 __ sub(as_Register($dst$$reg),
9223 as_Register($src1$$reg),
9224 as_Register($src2$$reg));
9225 %}
9226
9227 ins_pipe(ialu_reg_reg);
9228 %}
9229
9230 // No constant pool entries requiredLong Immediate Subtraction.
9231 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9232 match(Set dst (SubL src1 src2));
9233
9234 ins_cost(INSN_COST);
9235 format %{ "sub$dst, $src1, $src2" %}
9236
9237 // use opcode to indicate that this is a sub not an add
9238 opcode(0x1);
9239
9240 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9241
9242 ins_pipe(ialu_reg_imm);
9243 %}
9244
9245 // Integer Negation (special case for sub)
9246
9247 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9248 match(Set dst (SubI zero src));
9249
9250 ins_cost(INSN_COST);
9251 format %{ "negw $dst, $src\t# int" %}
9252
9253 ins_encode %{
9254 __ negw(as_Register($dst$$reg),
9255 as_Register($src$$reg));
9256 %}
9257
9258 ins_pipe(ialu_reg);
9259 %}
9260
9261 // Long Negation
9262
9263 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9264 match(Set dst (SubL zero src));
9265
9266 ins_cost(INSN_COST);
9267 format %{ "neg $dst, $src\t# long" %}
9268
9269 ins_encode %{
9270 __ neg(as_Register($dst$$reg),
9271 as_Register($src$$reg));
9272 %}
9273
9274 ins_pipe(ialu_reg);
9275 %}
9276
9277 // Integer Multiply
9278
9279 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9280 match(Set dst (MulI src1 src2));
9281
9282 ins_cost(INSN_COST * 3);
9283 format %{ "mulw $dst, $src1, $src2" %}
9284
9285 ins_encode %{
9286 __ mulw(as_Register($dst$$reg),
9287 as_Register($src1$$reg),
9288 as_Register($src2$$reg));
9289 %}
9290
9291 ins_pipe(imul_reg_reg);
9292 %}
9293
9294 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9295 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9296
9297 ins_cost(INSN_COST * 3);
9298 format %{ "smull $dst, $src1, $src2" %}
9299
9300 ins_encode %{
9301 __ smull(as_Register($dst$$reg),
9302 as_Register($src1$$reg),
9303 as_Register($src2$$reg));
9304 %}
9305
9306 ins_pipe(imul_reg_reg);
9307 %}
9308
9309 // Long Multiply
9310
9311 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9312 match(Set dst (MulL src1 src2));
9313
9314 ins_cost(INSN_COST * 5);
9315 format %{ "mul $dst, $src1, $src2" %}
9316
9317 ins_encode %{
9318 __ mul(as_Register($dst$$reg),
9319 as_Register($src1$$reg),
9320 as_Register($src2$$reg));
9321 %}
9322
9323 ins_pipe(lmul_reg_reg);
9324 %}
9325
9326 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9327 %{
9328 match(Set dst (MulHiL src1 src2));
9329
9330 ins_cost(INSN_COST * 7);
9331 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9332
9333 ins_encode %{
9334 __ smulh(as_Register($dst$$reg),
9335 as_Register($src1$$reg),
9336 as_Register($src2$$reg));
9337 %}
9338
9339 ins_pipe(lmul_reg_reg);
9340 %}
9341
9342 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9343 %{
9344 match(Set dst (UMulHiL src1 src2));
9345
9346 ins_cost(INSN_COST * 7);
9347 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9348
9349 ins_encode %{
9350 __ umulh(as_Register($dst$$reg),
9351 as_Register($src1$$reg),
9352 as_Register($src2$$reg));
9353 %}
9354
9355 ins_pipe(lmul_reg_reg);
9356 %}
9357
9358 // Combined Integer Multiply & Add/Sub
9359
9360 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9361 match(Set dst (AddI src3 (MulI src1 src2)));
9362
9363 ins_cost(INSN_COST * 3);
9364 format %{ "madd $dst, $src1, $src2, $src3" %}
9365
9366 ins_encode %{
9367 __ maddw(as_Register($dst$$reg),
9368 as_Register($src1$$reg),
9369 as_Register($src2$$reg),
9370 as_Register($src3$$reg));
9371 %}
9372
9373 ins_pipe(imac_reg_reg);
9374 %}
9375
9376 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9377 match(Set dst (SubI src3 (MulI src1 src2)));
9378
9379 ins_cost(INSN_COST * 3);
9380 format %{ "msub $dst, $src1, $src2, $src3" %}
9381
9382 ins_encode %{
9383 __ msubw(as_Register($dst$$reg),
9384 as_Register($src1$$reg),
9385 as_Register($src2$$reg),
9386 as_Register($src3$$reg));
9387 %}
9388
9389 ins_pipe(imac_reg_reg);
9390 %}
9391
9392 // Combined Integer Multiply & Neg
9393
9394 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9395 match(Set dst (MulI (SubI zero src1) src2));
9396
9397 ins_cost(INSN_COST * 3);
9398 format %{ "mneg $dst, $src1, $src2" %}
9399
9400 ins_encode %{
9401 __ mnegw(as_Register($dst$$reg),
9402 as_Register($src1$$reg),
9403 as_Register($src2$$reg));
9404 %}
9405
9406 ins_pipe(imac_reg_reg);
9407 %}
9408
9409 // Combined Long Multiply & Add/Sub
9410
9411 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9412 match(Set dst (AddL src3 (MulL src1 src2)));
9413
9414 ins_cost(INSN_COST * 5);
9415 format %{ "madd $dst, $src1, $src2, $src3" %}
9416
9417 ins_encode %{
9418 __ madd(as_Register($dst$$reg),
9419 as_Register($src1$$reg),
9420 as_Register($src2$$reg),
9421 as_Register($src3$$reg));
9422 %}
9423
9424 ins_pipe(lmac_reg_reg);
9425 %}
9426
9427 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9428 match(Set dst (SubL src3 (MulL src1 src2)));
9429
9430 ins_cost(INSN_COST * 5);
9431 format %{ "msub $dst, $src1, $src2, $src3" %}
9432
9433 ins_encode %{
9434 __ msub(as_Register($dst$$reg),
9435 as_Register($src1$$reg),
9436 as_Register($src2$$reg),
9437 as_Register($src3$$reg));
9438 %}
9439
9440 ins_pipe(lmac_reg_reg);
9441 %}
9442
9443 // Combined Long Multiply & Neg
9444
9445 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9446 match(Set dst (MulL (SubL zero src1) src2));
9447
9448 ins_cost(INSN_COST * 5);
9449 format %{ "mneg $dst, $src1, $src2" %}
9450
9451 ins_encode %{
9452 __ mneg(as_Register($dst$$reg),
9453 as_Register($src1$$reg),
9454 as_Register($src2$$reg));
9455 %}
9456
9457 ins_pipe(lmac_reg_reg);
9458 %}
9459
9460 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9461
9462 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9463 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9464
9465 ins_cost(INSN_COST * 3);
9466 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9467
9468 ins_encode %{
9469 __ smaddl(as_Register($dst$$reg),
9470 as_Register($src1$$reg),
9471 as_Register($src2$$reg),
9472 as_Register($src3$$reg));
9473 %}
9474
9475 ins_pipe(imac_reg_reg);
9476 %}
9477
9478 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9479 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9480
9481 ins_cost(INSN_COST * 3);
9482 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9483
9484 ins_encode %{
9485 __ smsubl(as_Register($dst$$reg),
9486 as_Register($src1$$reg),
9487 as_Register($src2$$reg),
9488 as_Register($src3$$reg));
9489 %}
9490
9491 ins_pipe(imac_reg_reg);
9492 %}
9493
9494 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9495 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9496
9497 ins_cost(INSN_COST * 3);
9498 format %{ "smnegl $dst, $src1, $src2" %}
9499
9500 ins_encode %{
9501 __ smnegl(as_Register($dst$$reg),
9502 as_Register($src1$$reg),
9503 as_Register($src2$$reg));
9504 %}
9505
9506 ins_pipe(imac_reg_reg);
9507 %}
9508
9509 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9510
9511 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9512 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9513
9514 ins_cost(INSN_COST * 5);
9515 format %{ "mulw rscratch1, $src1, $src2\n\t"
9516 "maddw $dst, $src3, $src4, rscratch1" %}
9517
9518 ins_encode %{
9519 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9520 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9521
9522 ins_pipe(imac_reg_reg);
9523 %}
9524
9525 // Integer Divide
9526
9527 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9528 match(Set dst (DivI src1 src2));
9529
9530 ins_cost(INSN_COST * 19);
9531 format %{ "sdivw $dst, $src1, $src2" %}
9532
9533 ins_encode(aarch64_enc_divw(dst, src1, src2));
9534 ins_pipe(idiv_reg_reg);
9535 %}
9536
9537 // Long Divide
9538
9539 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9540 match(Set dst (DivL src1 src2));
9541
9542 ins_cost(INSN_COST * 35);
9543 format %{ "sdiv $dst, $src1, $src2" %}
9544
9545 ins_encode(aarch64_enc_div(dst, src1, src2));
9546 ins_pipe(ldiv_reg_reg);
9547 %}
9548
9549 // Integer Remainder
9550
9551 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9552 match(Set dst (ModI src1 src2));
9553
9554 ins_cost(INSN_COST * 22);
9555 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9556 "msubw $dst, rscratch1, $src2, $src1" %}
9557
9558 ins_encode(aarch64_enc_modw(dst, src1, src2));
9559 ins_pipe(idiv_reg_reg);
9560 %}
9561
9562 // Long Remainder
9563
9564 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9565 match(Set dst (ModL src1 src2));
9566
9567 ins_cost(INSN_COST * 38);
9568 format %{ "sdiv rscratch1, $src1, $src2\n"
9569 "msub $dst, rscratch1, $src2, $src1" %}
9570
9571 ins_encode(aarch64_enc_mod(dst, src1, src2));
9572 ins_pipe(ldiv_reg_reg);
9573 %}
9574
9575 // Unsigned Integer Divide
9576
9577 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9578 match(Set dst (UDivI src1 src2));
9579
9580 ins_cost(INSN_COST * 19);
9581 format %{ "udivw $dst, $src1, $src2" %}
9582
9583 ins_encode %{
9584 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9585 %}
9586
9587 ins_pipe(idiv_reg_reg);
9588 %}
9589
9590 // Unsigned Long Divide
9591
9592 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9593 match(Set dst (UDivL src1 src2));
9594
9595 ins_cost(INSN_COST * 35);
9596 format %{ "udiv $dst, $src1, $src2" %}
9597
9598 ins_encode %{
9599 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9600 %}
9601
9602 ins_pipe(ldiv_reg_reg);
9603 %}
9604
9605 // Unsigned Integer Remainder
9606
9607 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9608 match(Set dst (UModI src1 src2));
9609
9610 ins_cost(INSN_COST * 22);
9611 format %{ "udivw rscratch1, $src1, $src2\n\t"
9612 "msubw $dst, rscratch1, $src2, $src1" %}
9613
9614 ins_encode %{
9615 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9616 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9617 %}
9618
9619 ins_pipe(idiv_reg_reg);
9620 %}
9621
9622 // Unsigned Long Remainder
9623
9624 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9625 match(Set dst (UModL src1 src2));
9626
9627 ins_cost(INSN_COST * 38);
9628 format %{ "udiv rscratch1, $src1, $src2\n"
9629 "msub $dst, rscratch1, $src2, $src1" %}
9630
9631 ins_encode %{
9632 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9633 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9634 %}
9635
9636 ins_pipe(ldiv_reg_reg);
9637 %}
9638
9639 // Integer Shifts
9640
9641 // Shift Left Register
9642 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9643 match(Set dst (LShiftI src1 src2));
9644
9645 ins_cost(INSN_COST * 2);
9646 format %{ "lslvw $dst, $src1, $src2" %}
9647
9648 ins_encode %{
9649 __ lslvw(as_Register($dst$$reg),
9650 as_Register($src1$$reg),
9651 as_Register($src2$$reg));
9652 %}
9653
9654 ins_pipe(ialu_reg_reg_vshift);
9655 %}
9656
9657 // Shift Left Immediate
9658 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9659 match(Set dst (LShiftI src1 src2));
9660
9661 ins_cost(INSN_COST);
9662 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9663
9664 ins_encode %{
9665 __ lslw(as_Register($dst$$reg),
9666 as_Register($src1$$reg),
9667 $src2$$constant & 0x1f);
9668 %}
9669
9670 ins_pipe(ialu_reg_shift);
9671 %}
9672
9673 // Shift Right Logical Register
9674 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9675 match(Set dst (URShiftI src1 src2));
9676
9677 ins_cost(INSN_COST * 2);
9678 format %{ "lsrvw $dst, $src1, $src2" %}
9679
9680 ins_encode %{
9681 __ lsrvw(as_Register($dst$$reg),
9682 as_Register($src1$$reg),
9683 as_Register($src2$$reg));
9684 %}
9685
9686 ins_pipe(ialu_reg_reg_vshift);
9687 %}
9688
9689 // Shift Right Logical Immediate
9690 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9691 match(Set dst (URShiftI src1 src2));
9692
9693 ins_cost(INSN_COST);
9694 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9695
9696 ins_encode %{
9697 __ lsrw(as_Register($dst$$reg),
9698 as_Register($src1$$reg),
9699 $src2$$constant & 0x1f);
9700 %}
9701
9702 ins_pipe(ialu_reg_shift);
9703 %}
9704
9705 // Shift Right Arithmetic Register
9706 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9707 match(Set dst (RShiftI src1 src2));
9708
9709 ins_cost(INSN_COST * 2);
9710 format %{ "asrvw $dst, $src1, $src2" %}
9711
9712 ins_encode %{
9713 __ asrvw(as_Register($dst$$reg),
9714 as_Register($src1$$reg),
9715 as_Register($src2$$reg));
9716 %}
9717
9718 ins_pipe(ialu_reg_reg_vshift);
9719 %}
9720
9721 // Shift Right Arithmetic Immediate
9722 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9723 match(Set dst (RShiftI src1 src2));
9724
9725 ins_cost(INSN_COST);
9726 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9727
9728 ins_encode %{
9729 __ asrw(as_Register($dst$$reg),
9730 as_Register($src1$$reg),
9731 $src2$$constant & 0x1f);
9732 %}
9733
9734 ins_pipe(ialu_reg_shift);
9735 %}
9736
9737 // Combined Int Mask and Right Shift (using UBFM)
9738 // TODO
9739
9740 // Long Shifts
9741
9742 // Shift Left Register
9743 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9744 match(Set dst (LShiftL src1 src2));
9745
9746 ins_cost(INSN_COST * 2);
9747 format %{ "lslv $dst, $src1, $src2" %}
9748
9749 ins_encode %{
9750 __ lslv(as_Register($dst$$reg),
9751 as_Register($src1$$reg),
9752 as_Register($src2$$reg));
9753 %}
9754
9755 ins_pipe(ialu_reg_reg_vshift);
9756 %}
9757
9758 // Shift Left Immediate
9759 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9760 match(Set dst (LShiftL src1 src2));
9761
9762 ins_cost(INSN_COST);
9763 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9764
9765 ins_encode %{
9766 __ lsl(as_Register($dst$$reg),
9767 as_Register($src1$$reg),
9768 $src2$$constant & 0x3f);
9769 %}
9770
9771 ins_pipe(ialu_reg_shift);
9772 %}
9773
9774 // Shift Right Logical Register
9775 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9776 match(Set dst (URShiftL src1 src2));
9777
9778 ins_cost(INSN_COST * 2);
9779 format %{ "lsrv $dst, $src1, $src2" %}
9780
9781 ins_encode %{
9782 __ lsrv(as_Register($dst$$reg),
9783 as_Register($src1$$reg),
9784 as_Register($src2$$reg));
9785 %}
9786
9787 ins_pipe(ialu_reg_reg_vshift);
9788 %}
9789
9790 // Shift Right Logical Immediate
9791 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9792 match(Set dst (URShiftL src1 src2));
9793
9794 ins_cost(INSN_COST);
9795 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9796
9797 ins_encode %{
9798 __ lsr(as_Register($dst$$reg),
9799 as_Register($src1$$reg),
9800 $src2$$constant & 0x3f);
9801 %}
9802
9803 ins_pipe(ialu_reg_shift);
9804 %}
9805
9806 // A special-case pattern for card table stores.
9807 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9808 match(Set dst (URShiftL (CastP2X src1) src2));
9809
9810 ins_cost(INSN_COST);
9811 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9812
9813 ins_encode %{
9814 __ lsr(as_Register($dst$$reg),
9815 as_Register($src1$$reg),
9816 $src2$$constant & 0x3f);
9817 %}
9818
9819 ins_pipe(ialu_reg_shift);
9820 %}
9821
9822 // Shift Right Arithmetic Register
9823 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9824 match(Set dst (RShiftL src1 src2));
9825
9826 ins_cost(INSN_COST * 2);
9827 format %{ "asrv $dst, $src1, $src2" %}
9828
9829 ins_encode %{
9830 __ asrv(as_Register($dst$$reg),
9831 as_Register($src1$$reg),
9832 as_Register($src2$$reg));
9833 %}
9834
9835 ins_pipe(ialu_reg_reg_vshift);
9836 %}
9837
9838 // Shift Right Arithmetic Immediate
9839 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9840 match(Set dst (RShiftL src1 src2));
9841
9842 ins_cost(INSN_COST);
9843 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9844
9845 ins_encode %{
9846 __ asr(as_Register($dst$$reg),
9847 as_Register($src1$$reg),
9848 $src2$$constant & 0x3f);
9849 %}
9850
9851 ins_pipe(ialu_reg_shift);
9852 %}
9853
9854 // BEGIN This section of the file is automatically generated. Do not edit --------------
9855 // This section is generated from aarch64_ad.m4
9856
9857 // This pattern is automatically generated from aarch64_ad.m4
9858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9859 instruct regL_not_reg(iRegLNoSp dst,
9860 iRegL src1, immL_M1 m1,
9861 rFlagsReg cr) %{
9862 match(Set dst (XorL src1 m1));
9863 ins_cost(INSN_COST);
9864 format %{ "eon $dst, $src1, zr" %}
9865
9866 ins_encode %{
9867 __ eon(as_Register($dst$$reg),
9868 as_Register($src1$$reg),
9869 zr,
9870 Assembler::LSL, 0);
9871 %}
9872
9873 ins_pipe(ialu_reg);
9874 %}
9875
9876 // This pattern is automatically generated from aarch64_ad.m4
9877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9878 instruct regI_not_reg(iRegINoSp dst,
9879 iRegIorL2I src1, immI_M1 m1,
9880 rFlagsReg cr) %{
9881 match(Set dst (XorI src1 m1));
9882 ins_cost(INSN_COST);
9883 format %{ "eonw $dst, $src1, zr" %}
9884
9885 ins_encode %{
9886 __ eonw(as_Register($dst$$reg),
9887 as_Register($src1$$reg),
9888 zr,
9889 Assembler::LSL, 0);
9890 %}
9891
9892 ins_pipe(ialu_reg);
9893 %}
9894
9895 // This pattern is automatically generated from aarch64_ad.m4
9896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9897 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9898 immI0 zero, iRegIorL2I src1, immI src2) %{
9899 match(Set dst (SubI zero (URShiftI src1 src2)));
9900
9901 ins_cost(1.9 * INSN_COST);
9902 format %{ "negw $dst, $src1, LSR $src2" %}
9903
9904 ins_encode %{
9905 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9906 Assembler::LSR, $src2$$constant & 0x1f);
9907 %}
9908
9909 ins_pipe(ialu_reg_shift);
9910 %}
9911
9912 // This pattern is automatically generated from aarch64_ad.m4
9913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9914 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9915 immI0 zero, iRegIorL2I src1, immI src2) %{
9916 match(Set dst (SubI zero (RShiftI src1 src2)));
9917
9918 ins_cost(1.9 * INSN_COST);
9919 format %{ "negw $dst, $src1, ASR $src2" %}
9920
9921 ins_encode %{
9922 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9923 Assembler::ASR, $src2$$constant & 0x1f);
9924 %}
9925
9926 ins_pipe(ialu_reg_shift);
9927 %}
9928
9929 // This pattern is automatically generated from aarch64_ad.m4
9930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9931 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9932 immI0 zero, iRegIorL2I src1, immI src2) %{
9933 match(Set dst (SubI zero (LShiftI src1 src2)));
9934
9935 ins_cost(1.9 * INSN_COST);
9936 format %{ "negw $dst, $src1, LSL $src2" %}
9937
9938 ins_encode %{
9939 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9940 Assembler::LSL, $src2$$constant & 0x1f);
9941 %}
9942
9943 ins_pipe(ialu_reg_shift);
9944 %}
9945
9946 // This pattern is automatically generated from aarch64_ad.m4
9947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9948 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9949 immL0 zero, iRegL src1, immI src2) %{
9950 match(Set dst (SubL zero (URShiftL src1 src2)));
9951
9952 ins_cost(1.9 * INSN_COST);
9953 format %{ "neg $dst, $src1, LSR $src2" %}
9954
9955 ins_encode %{
9956 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9957 Assembler::LSR, $src2$$constant & 0x3f);
9958 %}
9959
9960 ins_pipe(ialu_reg_shift);
9961 %}
9962
9963 // This pattern is automatically generated from aarch64_ad.m4
9964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9965 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9966 immL0 zero, iRegL src1, immI src2) %{
9967 match(Set dst (SubL zero (RShiftL src1 src2)));
9968
9969 ins_cost(1.9 * INSN_COST);
9970 format %{ "neg $dst, $src1, ASR $src2" %}
9971
9972 ins_encode %{
9973 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9974 Assembler::ASR, $src2$$constant & 0x3f);
9975 %}
9976
9977 ins_pipe(ialu_reg_shift);
9978 %}
9979
9980 // This pattern is automatically generated from aarch64_ad.m4
9981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9982 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9983 immL0 zero, iRegL src1, immI src2) %{
9984 match(Set dst (SubL zero (LShiftL src1 src2)));
9985
9986 ins_cost(1.9 * INSN_COST);
9987 format %{ "neg $dst, $src1, LSL $src2" %}
9988
9989 ins_encode %{
9990 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9991 Assembler::LSL, $src2$$constant & 0x3f);
9992 %}
9993
9994 ins_pipe(ialu_reg_shift);
9995 %}
9996
9997 // This pattern is automatically generated from aarch64_ad.m4
9998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9999 instruct AndI_reg_not_reg(iRegINoSp dst,
10000 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10001 match(Set dst (AndI src1 (XorI src2 m1)));
10002 ins_cost(INSN_COST);
10003 format %{ "bicw $dst, $src1, $src2" %}
10004
10005 ins_encode %{
10006 __ bicw(as_Register($dst$$reg),
10007 as_Register($src1$$reg),
10008 as_Register($src2$$reg),
10009 Assembler::LSL, 0);
10010 %}
10011
10012 ins_pipe(ialu_reg_reg);
10013 %}
10014
10015 // This pattern is automatically generated from aarch64_ad.m4
10016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10017 instruct AndL_reg_not_reg(iRegLNoSp dst,
10018 iRegL src1, iRegL src2, immL_M1 m1) %{
10019 match(Set dst (AndL src1 (XorL src2 m1)));
10020 ins_cost(INSN_COST);
10021 format %{ "bic $dst, $src1, $src2" %}
10022
10023 ins_encode %{
10024 __ bic(as_Register($dst$$reg),
10025 as_Register($src1$$reg),
10026 as_Register($src2$$reg),
10027 Assembler::LSL, 0);
10028 %}
10029
10030 ins_pipe(ialu_reg_reg);
10031 %}
10032
10033 // This pattern is automatically generated from aarch64_ad.m4
10034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10035 instruct OrI_reg_not_reg(iRegINoSp dst,
10036 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10037 match(Set dst (OrI src1 (XorI src2 m1)));
10038 ins_cost(INSN_COST);
10039 format %{ "ornw $dst, $src1, $src2" %}
10040
10041 ins_encode %{
10042 __ ornw(as_Register($dst$$reg),
10043 as_Register($src1$$reg),
10044 as_Register($src2$$reg),
10045 Assembler::LSL, 0);
10046 %}
10047
10048 ins_pipe(ialu_reg_reg);
10049 %}
10050
10051 // This pattern is automatically generated from aarch64_ad.m4
10052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10053 instruct OrL_reg_not_reg(iRegLNoSp dst,
10054 iRegL src1, iRegL src2, immL_M1 m1) %{
10055 match(Set dst (OrL src1 (XorL src2 m1)));
10056 ins_cost(INSN_COST);
10057 format %{ "orn $dst, $src1, $src2" %}
10058
10059 ins_encode %{
10060 __ orn(as_Register($dst$$reg),
10061 as_Register($src1$$reg),
10062 as_Register($src2$$reg),
10063 Assembler::LSL, 0);
10064 %}
10065
10066 ins_pipe(ialu_reg_reg);
10067 %}
10068
10069 // This pattern is automatically generated from aarch64_ad.m4
10070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10071 instruct XorI_reg_not_reg(iRegINoSp dst,
10072 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10073 match(Set dst (XorI m1 (XorI src2 src1)));
10074 ins_cost(INSN_COST);
10075 format %{ "eonw $dst, $src1, $src2" %}
10076
10077 ins_encode %{
10078 __ eonw(as_Register($dst$$reg),
10079 as_Register($src1$$reg),
10080 as_Register($src2$$reg),
10081 Assembler::LSL, 0);
10082 %}
10083
10084 ins_pipe(ialu_reg_reg);
10085 %}
10086
10087 // This pattern is automatically generated from aarch64_ad.m4
10088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10089 instruct XorL_reg_not_reg(iRegLNoSp dst,
10090 iRegL src1, iRegL src2, immL_M1 m1) %{
10091 match(Set dst (XorL m1 (XorL src2 src1)));
10092 ins_cost(INSN_COST);
10093 format %{ "eon $dst, $src1, $src2" %}
10094
10095 ins_encode %{
10096 __ eon(as_Register($dst$$reg),
10097 as_Register($src1$$reg),
10098 as_Register($src2$$reg),
10099 Assembler::LSL, 0);
10100 %}
10101
10102 ins_pipe(ialu_reg_reg);
10103 %}
10104
10105 // This pattern is automatically generated from aarch64_ad.m4
10106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10107 // val & (-1 ^ (val >>> shift)) ==> bicw
10108 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10109 iRegIorL2I src1, iRegIorL2I src2,
10110 immI src3, immI_M1 src4) %{
10111 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10112 ins_cost(1.9 * INSN_COST);
10113 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10114
10115 ins_encode %{
10116 __ bicw(as_Register($dst$$reg),
10117 as_Register($src1$$reg),
10118 as_Register($src2$$reg),
10119 Assembler::LSR,
10120 $src3$$constant & 0x1f);
10121 %}
10122
10123 ins_pipe(ialu_reg_reg_shift);
10124 %}
10125
10126 // This pattern is automatically generated from aarch64_ad.m4
10127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10128 // val & (-1 ^ (val >>> shift)) ==> bic
10129 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10130 iRegL src1, iRegL src2,
10131 immI src3, immL_M1 src4) %{
10132 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10133 ins_cost(1.9 * INSN_COST);
10134 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10135
10136 ins_encode %{
10137 __ bic(as_Register($dst$$reg),
10138 as_Register($src1$$reg),
10139 as_Register($src2$$reg),
10140 Assembler::LSR,
10141 $src3$$constant & 0x3f);
10142 %}
10143
10144 ins_pipe(ialu_reg_reg_shift);
10145 %}
10146
10147 // This pattern is automatically generated from aarch64_ad.m4
10148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10149 // val & (-1 ^ (val >> shift)) ==> bicw
10150 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10151 iRegIorL2I src1, iRegIorL2I src2,
10152 immI src3, immI_M1 src4) %{
10153 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10154 ins_cost(1.9 * INSN_COST);
10155 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10156
10157 ins_encode %{
10158 __ bicw(as_Register($dst$$reg),
10159 as_Register($src1$$reg),
10160 as_Register($src2$$reg),
10161 Assembler::ASR,
10162 $src3$$constant & 0x1f);
10163 %}
10164
10165 ins_pipe(ialu_reg_reg_shift);
10166 %}
10167
10168 // This pattern is automatically generated from aarch64_ad.m4
10169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10170 // val & (-1 ^ (val >> shift)) ==> bic
10171 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10172 iRegL src1, iRegL src2,
10173 immI src3, immL_M1 src4) %{
10174 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10175 ins_cost(1.9 * INSN_COST);
10176 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10177
10178 ins_encode %{
10179 __ bic(as_Register($dst$$reg),
10180 as_Register($src1$$reg),
10181 as_Register($src2$$reg),
10182 Assembler::ASR,
10183 $src3$$constant & 0x3f);
10184 %}
10185
10186 ins_pipe(ialu_reg_reg_shift);
10187 %}
10188
10189 // This pattern is automatically generated from aarch64_ad.m4
10190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10191 // val & (-1 ^ (val ror shift)) ==> bicw
10192 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10193 iRegIorL2I src1, iRegIorL2I src2,
10194 immI src3, immI_M1 src4) %{
10195 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10196 ins_cost(1.9 * INSN_COST);
10197 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10198
10199 ins_encode %{
10200 __ bicw(as_Register($dst$$reg),
10201 as_Register($src1$$reg),
10202 as_Register($src2$$reg),
10203 Assembler::ROR,
10204 $src3$$constant & 0x1f);
10205 %}
10206
10207 ins_pipe(ialu_reg_reg_shift);
10208 %}
10209
10210 // This pattern is automatically generated from aarch64_ad.m4
10211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10212 // val & (-1 ^ (val ror shift)) ==> bic
10213 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10214 iRegL src1, iRegL src2,
10215 immI src3, immL_M1 src4) %{
10216 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10217 ins_cost(1.9 * INSN_COST);
10218 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10219
10220 ins_encode %{
10221 __ bic(as_Register($dst$$reg),
10222 as_Register($src1$$reg),
10223 as_Register($src2$$reg),
10224 Assembler::ROR,
10225 $src3$$constant & 0x3f);
10226 %}
10227
10228 ins_pipe(ialu_reg_reg_shift);
10229 %}
10230
10231 // This pattern is automatically generated from aarch64_ad.m4
10232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10233 // val & (-1 ^ (val << shift)) ==> bicw
10234 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10235 iRegIorL2I src1, iRegIorL2I src2,
10236 immI src3, immI_M1 src4) %{
10237 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10238 ins_cost(1.9 * INSN_COST);
10239 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10240
10241 ins_encode %{
10242 __ bicw(as_Register($dst$$reg),
10243 as_Register($src1$$reg),
10244 as_Register($src2$$reg),
10245 Assembler::LSL,
10246 $src3$$constant & 0x1f);
10247 %}
10248
10249 ins_pipe(ialu_reg_reg_shift);
10250 %}
10251
10252 // This pattern is automatically generated from aarch64_ad.m4
10253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10254 // val & (-1 ^ (val << shift)) ==> bic
10255 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10256 iRegL src1, iRegL src2,
10257 immI src3, immL_M1 src4) %{
10258 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10259 ins_cost(1.9 * INSN_COST);
10260 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10261
10262 ins_encode %{
10263 __ bic(as_Register($dst$$reg),
10264 as_Register($src1$$reg),
10265 as_Register($src2$$reg),
10266 Assembler::LSL,
10267 $src3$$constant & 0x3f);
10268 %}
10269
10270 ins_pipe(ialu_reg_reg_shift);
10271 %}
10272
10273 // This pattern is automatically generated from aarch64_ad.m4
10274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10275 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10276 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10277 iRegIorL2I src1, iRegIorL2I src2,
10278 immI src3, immI_M1 src4) %{
10279 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10280 ins_cost(1.9 * INSN_COST);
10281 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10282
10283 ins_encode %{
10284 __ eonw(as_Register($dst$$reg),
10285 as_Register($src1$$reg),
10286 as_Register($src2$$reg),
10287 Assembler::LSR,
10288 $src3$$constant & 0x1f);
10289 %}
10290
10291 ins_pipe(ialu_reg_reg_shift);
10292 %}
10293
10294 // This pattern is automatically generated from aarch64_ad.m4
10295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10296 // val ^ (-1 ^ (val >>> shift)) ==> eon
10297 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10298 iRegL src1, iRegL src2,
10299 immI src3, immL_M1 src4) %{
10300 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10301 ins_cost(1.9 * INSN_COST);
10302 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10303
10304 ins_encode %{
10305 __ eon(as_Register($dst$$reg),
10306 as_Register($src1$$reg),
10307 as_Register($src2$$reg),
10308 Assembler::LSR,
10309 $src3$$constant & 0x3f);
10310 %}
10311
10312 ins_pipe(ialu_reg_reg_shift);
10313 %}
10314
10315 // This pattern is automatically generated from aarch64_ad.m4
10316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10317 // val ^ (-1 ^ (val >> shift)) ==> eonw
10318 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10319 iRegIorL2I src1, iRegIorL2I src2,
10320 immI src3, immI_M1 src4) %{
10321 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10322 ins_cost(1.9 * INSN_COST);
10323 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10324
10325 ins_encode %{
10326 __ eonw(as_Register($dst$$reg),
10327 as_Register($src1$$reg),
10328 as_Register($src2$$reg),
10329 Assembler::ASR,
10330 $src3$$constant & 0x1f);
10331 %}
10332
10333 ins_pipe(ialu_reg_reg_shift);
10334 %}
10335
10336 // This pattern is automatically generated from aarch64_ad.m4
10337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10338 // val ^ (-1 ^ (val >> shift)) ==> eon
10339 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10340 iRegL src1, iRegL src2,
10341 immI src3, immL_M1 src4) %{
10342 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10343 ins_cost(1.9 * INSN_COST);
10344 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10345
10346 ins_encode %{
10347 __ eon(as_Register($dst$$reg),
10348 as_Register($src1$$reg),
10349 as_Register($src2$$reg),
10350 Assembler::ASR,
10351 $src3$$constant & 0x3f);
10352 %}
10353
10354 ins_pipe(ialu_reg_reg_shift);
10355 %}
10356
10357 // This pattern is automatically generated from aarch64_ad.m4
10358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10359 // val ^ (-1 ^ (val ror shift)) ==> eonw
10360 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10361 iRegIorL2I src1, iRegIorL2I src2,
10362 immI src3, immI_M1 src4) %{
10363 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10364 ins_cost(1.9 * INSN_COST);
10365 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10366
10367 ins_encode %{
10368 __ eonw(as_Register($dst$$reg),
10369 as_Register($src1$$reg),
10370 as_Register($src2$$reg),
10371 Assembler::ROR,
10372 $src3$$constant & 0x1f);
10373 %}
10374
10375 ins_pipe(ialu_reg_reg_shift);
10376 %}
10377
10378 // This pattern is automatically generated from aarch64_ad.m4
10379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10380 // val ^ (-1 ^ (val ror shift)) ==> eon
10381 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10382 iRegL src1, iRegL src2,
10383 immI src3, immL_M1 src4) %{
10384 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10385 ins_cost(1.9 * INSN_COST);
10386 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10387
10388 ins_encode %{
10389 __ eon(as_Register($dst$$reg),
10390 as_Register($src1$$reg),
10391 as_Register($src2$$reg),
10392 Assembler::ROR,
10393 $src3$$constant & 0x3f);
10394 %}
10395
10396 ins_pipe(ialu_reg_reg_shift);
10397 %}
10398
10399 // This pattern is automatically generated from aarch64_ad.m4
10400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10401 // val ^ (-1 ^ (val << shift)) ==> eonw
10402 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10403 iRegIorL2I src1, iRegIorL2I src2,
10404 immI src3, immI_M1 src4) %{
10405 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10406 ins_cost(1.9 * INSN_COST);
10407 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10408
10409 ins_encode %{
10410 __ eonw(as_Register($dst$$reg),
10411 as_Register($src1$$reg),
10412 as_Register($src2$$reg),
10413 Assembler::LSL,
10414 $src3$$constant & 0x1f);
10415 %}
10416
10417 ins_pipe(ialu_reg_reg_shift);
10418 %}
10419
10420 // This pattern is automatically generated from aarch64_ad.m4
10421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10422 // val ^ (-1 ^ (val << shift)) ==> eon
10423 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10424 iRegL src1, iRegL src2,
10425 immI src3, immL_M1 src4) %{
10426 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10427 ins_cost(1.9 * INSN_COST);
10428 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10429
10430 ins_encode %{
10431 __ eon(as_Register($dst$$reg),
10432 as_Register($src1$$reg),
10433 as_Register($src2$$reg),
10434 Assembler::LSL,
10435 $src3$$constant & 0x3f);
10436 %}
10437
10438 ins_pipe(ialu_reg_reg_shift);
10439 %}
10440
10441 // This pattern is automatically generated from aarch64_ad.m4
10442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10443 // val | (-1 ^ (val >>> shift)) ==> ornw
10444 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10445 iRegIorL2I src1, iRegIorL2I src2,
10446 immI src3, immI_M1 src4) %{
10447 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10448 ins_cost(1.9 * INSN_COST);
10449 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10450
10451 ins_encode %{
10452 __ ornw(as_Register($dst$$reg),
10453 as_Register($src1$$reg),
10454 as_Register($src2$$reg),
10455 Assembler::LSR,
10456 $src3$$constant & 0x1f);
10457 %}
10458
10459 ins_pipe(ialu_reg_reg_shift);
10460 %}
10461
10462 // This pattern is automatically generated from aarch64_ad.m4
10463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10464 // val | (-1 ^ (val >>> shift)) ==> orn
10465 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10466 iRegL src1, iRegL src2,
10467 immI src3, immL_M1 src4) %{
10468 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10469 ins_cost(1.9 * INSN_COST);
10470 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10471
10472 ins_encode %{
10473 __ orn(as_Register($dst$$reg),
10474 as_Register($src1$$reg),
10475 as_Register($src2$$reg),
10476 Assembler::LSR,
10477 $src3$$constant & 0x3f);
10478 %}
10479
10480 ins_pipe(ialu_reg_reg_shift);
10481 %}
10482
10483 // This pattern is automatically generated from aarch64_ad.m4
10484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10485 // val | (-1 ^ (val >> shift)) ==> ornw
10486 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10487 iRegIorL2I src1, iRegIorL2I src2,
10488 immI src3, immI_M1 src4) %{
10489 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10490 ins_cost(1.9 * INSN_COST);
10491 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10492
10493 ins_encode %{
10494 __ ornw(as_Register($dst$$reg),
10495 as_Register($src1$$reg),
10496 as_Register($src2$$reg),
10497 Assembler::ASR,
10498 $src3$$constant & 0x1f);
10499 %}
10500
10501 ins_pipe(ialu_reg_reg_shift);
10502 %}
10503
10504 // This pattern is automatically generated from aarch64_ad.m4
10505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10506 // val | (-1 ^ (val >> shift)) ==> orn
10507 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10508 iRegL src1, iRegL src2,
10509 immI src3, immL_M1 src4) %{
10510 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10511 ins_cost(1.9 * INSN_COST);
10512 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10513
10514 ins_encode %{
10515 __ orn(as_Register($dst$$reg),
10516 as_Register($src1$$reg),
10517 as_Register($src2$$reg),
10518 Assembler::ASR,
10519 $src3$$constant & 0x3f);
10520 %}
10521
10522 ins_pipe(ialu_reg_reg_shift);
10523 %}
10524
10525 // This pattern is automatically generated from aarch64_ad.m4
10526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10527 // val | (-1 ^ (val ror shift)) ==> ornw
10528 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10529 iRegIorL2I src1, iRegIorL2I src2,
10530 immI src3, immI_M1 src4) %{
10531 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10532 ins_cost(1.9 * INSN_COST);
10533 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10534
10535 ins_encode %{
10536 __ ornw(as_Register($dst$$reg),
10537 as_Register($src1$$reg),
10538 as_Register($src2$$reg),
10539 Assembler::ROR,
10540 $src3$$constant & 0x1f);
10541 %}
10542
10543 ins_pipe(ialu_reg_reg_shift);
10544 %}
10545
10546 // This pattern is automatically generated from aarch64_ad.m4
10547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10548 // val | (-1 ^ (val ror shift)) ==> orn
10549 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10550 iRegL src1, iRegL src2,
10551 immI src3, immL_M1 src4) %{
10552 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10553 ins_cost(1.9 * INSN_COST);
10554 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10555
10556 ins_encode %{
10557 __ orn(as_Register($dst$$reg),
10558 as_Register($src1$$reg),
10559 as_Register($src2$$reg),
10560 Assembler::ROR,
10561 $src3$$constant & 0x3f);
10562 %}
10563
10564 ins_pipe(ialu_reg_reg_shift);
10565 %}
10566
10567 // This pattern is automatically generated from aarch64_ad.m4
10568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10569 // val | (-1 ^ (val << shift)) ==> ornw
10570 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10571 iRegIorL2I src1, iRegIorL2I src2,
10572 immI src3, immI_M1 src4) %{
10573 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10574 ins_cost(1.9 * INSN_COST);
10575 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10576
10577 ins_encode %{
10578 __ ornw(as_Register($dst$$reg),
10579 as_Register($src1$$reg),
10580 as_Register($src2$$reg),
10581 Assembler::LSL,
10582 $src3$$constant & 0x1f);
10583 %}
10584
10585 ins_pipe(ialu_reg_reg_shift);
10586 %}
10587
10588 // This pattern is automatically generated from aarch64_ad.m4
10589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10590 // val | (-1 ^ (val << shift)) ==> orn
10591 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10592 iRegL src1, iRegL src2,
10593 immI src3, immL_M1 src4) %{
10594 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10595 ins_cost(1.9 * INSN_COST);
10596 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10597
10598 ins_encode %{
10599 __ orn(as_Register($dst$$reg),
10600 as_Register($src1$$reg),
10601 as_Register($src2$$reg),
10602 Assembler::LSL,
10603 $src3$$constant & 0x3f);
10604 %}
10605
10606 ins_pipe(ialu_reg_reg_shift);
10607 %}
10608
10609 // This pattern is automatically generated from aarch64_ad.m4
10610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10611 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10612 iRegIorL2I src1, iRegIorL2I src2,
10613 immI src3) %{
10614 match(Set dst (AndI src1 (URShiftI src2 src3)));
10615
10616 ins_cost(1.9 * INSN_COST);
10617 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10618
10619 ins_encode %{
10620 __ andw(as_Register($dst$$reg),
10621 as_Register($src1$$reg),
10622 as_Register($src2$$reg),
10623 Assembler::LSR,
10624 $src3$$constant & 0x1f);
10625 %}
10626
10627 ins_pipe(ialu_reg_reg_shift);
10628 %}
10629
10630 // This pattern is automatically generated from aarch64_ad.m4
10631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10632 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10633 iRegL src1, iRegL src2,
10634 immI src3) %{
10635 match(Set dst (AndL src1 (URShiftL src2 src3)));
10636
10637 ins_cost(1.9 * INSN_COST);
10638 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10639
10640 ins_encode %{
10641 __ andr(as_Register($dst$$reg),
10642 as_Register($src1$$reg),
10643 as_Register($src2$$reg),
10644 Assembler::LSR,
10645 $src3$$constant & 0x3f);
10646 %}
10647
10648 ins_pipe(ialu_reg_reg_shift);
10649 %}
10650
10651 // This pattern is automatically generated from aarch64_ad.m4
10652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10653 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10654 iRegIorL2I src1, iRegIorL2I src2,
10655 immI src3) %{
10656 match(Set dst (AndI src1 (RShiftI src2 src3)));
10657
10658 ins_cost(1.9 * INSN_COST);
10659 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10660
10661 ins_encode %{
10662 __ andw(as_Register($dst$$reg),
10663 as_Register($src1$$reg),
10664 as_Register($src2$$reg),
10665 Assembler::ASR,
10666 $src3$$constant & 0x1f);
10667 %}
10668
10669 ins_pipe(ialu_reg_reg_shift);
10670 %}
10671
10672 // This pattern is automatically generated from aarch64_ad.m4
10673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10674 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10675 iRegL src1, iRegL src2,
10676 immI src3) %{
10677 match(Set dst (AndL src1 (RShiftL src2 src3)));
10678
10679 ins_cost(1.9 * INSN_COST);
10680 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10681
10682 ins_encode %{
10683 __ andr(as_Register($dst$$reg),
10684 as_Register($src1$$reg),
10685 as_Register($src2$$reg),
10686 Assembler::ASR,
10687 $src3$$constant & 0x3f);
10688 %}
10689
10690 ins_pipe(ialu_reg_reg_shift);
10691 %}
10692
10693 // This pattern is automatically generated from aarch64_ad.m4
10694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10695 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10696 iRegIorL2I src1, iRegIorL2I src2,
10697 immI src3) %{
10698 match(Set dst (AndI src1 (LShiftI src2 src3)));
10699
10700 ins_cost(1.9 * INSN_COST);
10701 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10702
10703 ins_encode %{
10704 __ andw(as_Register($dst$$reg),
10705 as_Register($src1$$reg),
10706 as_Register($src2$$reg),
10707 Assembler::LSL,
10708 $src3$$constant & 0x1f);
10709 %}
10710
10711 ins_pipe(ialu_reg_reg_shift);
10712 %}
10713
10714 // This pattern is automatically generated from aarch64_ad.m4
10715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10716 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10717 iRegL src1, iRegL src2,
10718 immI src3) %{
10719 match(Set dst (AndL src1 (LShiftL src2 src3)));
10720
10721 ins_cost(1.9 * INSN_COST);
10722 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10723
10724 ins_encode %{
10725 __ andr(as_Register($dst$$reg),
10726 as_Register($src1$$reg),
10727 as_Register($src2$$reg),
10728 Assembler::LSL,
10729 $src3$$constant & 0x3f);
10730 %}
10731
10732 ins_pipe(ialu_reg_reg_shift);
10733 %}
10734
10735 // This pattern is automatically generated from aarch64_ad.m4
10736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10737 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10738 iRegIorL2I src1, iRegIorL2I src2,
10739 immI src3) %{
10740 match(Set dst (AndI src1 (RotateRight src2 src3)));
10741
10742 ins_cost(1.9 * INSN_COST);
10743 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10744
10745 ins_encode %{
10746 __ andw(as_Register($dst$$reg),
10747 as_Register($src1$$reg),
10748 as_Register($src2$$reg),
10749 Assembler::ROR,
10750 $src3$$constant & 0x1f);
10751 %}
10752
10753 ins_pipe(ialu_reg_reg_shift);
10754 %}
10755
10756 // This pattern is automatically generated from aarch64_ad.m4
10757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10758 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10759 iRegL src1, iRegL src2,
10760 immI src3) %{
10761 match(Set dst (AndL src1 (RotateRight src2 src3)));
10762
10763 ins_cost(1.9 * INSN_COST);
10764 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10765
10766 ins_encode %{
10767 __ andr(as_Register($dst$$reg),
10768 as_Register($src1$$reg),
10769 as_Register($src2$$reg),
10770 Assembler::ROR,
10771 $src3$$constant & 0x3f);
10772 %}
10773
10774 ins_pipe(ialu_reg_reg_shift);
10775 %}
10776
10777 // This pattern is automatically generated from aarch64_ad.m4
10778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10779 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10780 iRegIorL2I src1, iRegIorL2I src2,
10781 immI src3) %{
10782 match(Set dst (XorI src1 (URShiftI src2 src3)));
10783
10784 ins_cost(1.9 * INSN_COST);
10785 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10786
10787 ins_encode %{
10788 __ eorw(as_Register($dst$$reg),
10789 as_Register($src1$$reg),
10790 as_Register($src2$$reg),
10791 Assembler::LSR,
10792 $src3$$constant & 0x1f);
10793 %}
10794
10795 ins_pipe(ialu_reg_reg_shift);
10796 %}
10797
10798 // This pattern is automatically generated from aarch64_ad.m4
10799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10800 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10801 iRegL src1, iRegL src2,
10802 immI src3) %{
10803 match(Set dst (XorL src1 (URShiftL src2 src3)));
10804
10805 ins_cost(1.9 * INSN_COST);
10806 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10807
10808 ins_encode %{
10809 __ eor(as_Register($dst$$reg),
10810 as_Register($src1$$reg),
10811 as_Register($src2$$reg),
10812 Assembler::LSR,
10813 $src3$$constant & 0x3f);
10814 %}
10815
10816 ins_pipe(ialu_reg_reg_shift);
10817 %}
10818
10819 // This pattern is automatically generated from aarch64_ad.m4
10820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10821 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10822 iRegIorL2I src1, iRegIorL2I src2,
10823 immI src3) %{
10824 match(Set dst (XorI src1 (RShiftI src2 src3)));
10825
10826 ins_cost(1.9 * INSN_COST);
10827 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10828
10829 ins_encode %{
10830 __ eorw(as_Register($dst$$reg),
10831 as_Register($src1$$reg),
10832 as_Register($src2$$reg),
10833 Assembler::ASR,
10834 $src3$$constant & 0x1f);
10835 %}
10836
10837 ins_pipe(ialu_reg_reg_shift);
10838 %}
10839
10840 // This pattern is automatically generated from aarch64_ad.m4
10841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10842 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10843 iRegL src1, iRegL src2,
10844 immI src3) %{
10845 match(Set dst (XorL src1 (RShiftL src2 src3)));
10846
10847 ins_cost(1.9 * INSN_COST);
10848 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10849
10850 ins_encode %{
10851 __ eor(as_Register($dst$$reg),
10852 as_Register($src1$$reg),
10853 as_Register($src2$$reg),
10854 Assembler::ASR,
10855 $src3$$constant & 0x3f);
10856 %}
10857
10858 ins_pipe(ialu_reg_reg_shift);
10859 %}
10860
10861 // This pattern is automatically generated from aarch64_ad.m4
10862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10863 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10864 iRegIorL2I src1, iRegIorL2I src2,
10865 immI src3) %{
10866 match(Set dst (XorI src1 (LShiftI src2 src3)));
10867
10868 ins_cost(1.9 * INSN_COST);
10869 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10870
10871 ins_encode %{
10872 __ eorw(as_Register($dst$$reg),
10873 as_Register($src1$$reg),
10874 as_Register($src2$$reg),
10875 Assembler::LSL,
10876 $src3$$constant & 0x1f);
10877 %}
10878
10879 ins_pipe(ialu_reg_reg_shift);
10880 %}
10881
10882 // This pattern is automatically generated from aarch64_ad.m4
10883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10884 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10885 iRegL src1, iRegL src2,
10886 immI src3) %{
10887 match(Set dst (XorL src1 (LShiftL src2 src3)));
10888
10889 ins_cost(1.9 * INSN_COST);
10890 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10891
10892 ins_encode %{
10893 __ eor(as_Register($dst$$reg),
10894 as_Register($src1$$reg),
10895 as_Register($src2$$reg),
10896 Assembler::LSL,
10897 $src3$$constant & 0x3f);
10898 %}
10899
10900 ins_pipe(ialu_reg_reg_shift);
10901 %}
10902
10903 // This pattern is automatically generated from aarch64_ad.m4
10904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10905 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10906 iRegIorL2I src1, iRegIorL2I src2,
10907 immI src3) %{
10908 match(Set dst (XorI src1 (RotateRight src2 src3)));
10909
10910 ins_cost(1.9 * INSN_COST);
10911 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10912
10913 ins_encode %{
10914 __ eorw(as_Register($dst$$reg),
10915 as_Register($src1$$reg),
10916 as_Register($src2$$reg),
10917 Assembler::ROR,
10918 $src3$$constant & 0x1f);
10919 %}
10920
10921 ins_pipe(ialu_reg_reg_shift);
10922 %}
10923
10924 // This pattern is automatically generated from aarch64_ad.m4
10925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10926 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10927 iRegL src1, iRegL src2,
10928 immI src3) %{
10929 match(Set dst (XorL src1 (RotateRight src2 src3)));
10930
10931 ins_cost(1.9 * INSN_COST);
10932 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10933
10934 ins_encode %{
10935 __ eor(as_Register($dst$$reg),
10936 as_Register($src1$$reg),
10937 as_Register($src2$$reg),
10938 Assembler::ROR,
10939 $src3$$constant & 0x3f);
10940 %}
10941
10942 ins_pipe(ialu_reg_reg_shift);
10943 %}
10944
10945 // This pattern is automatically generated from aarch64_ad.m4
10946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10947 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10948 iRegIorL2I src1, iRegIorL2I src2,
10949 immI src3) %{
10950 match(Set dst (OrI src1 (URShiftI src2 src3)));
10951
10952 ins_cost(1.9 * INSN_COST);
10953 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10954
10955 ins_encode %{
10956 __ orrw(as_Register($dst$$reg),
10957 as_Register($src1$$reg),
10958 as_Register($src2$$reg),
10959 Assembler::LSR,
10960 $src3$$constant & 0x1f);
10961 %}
10962
10963 ins_pipe(ialu_reg_reg_shift);
10964 %}
10965
10966 // This pattern is automatically generated from aarch64_ad.m4
10967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10968 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10969 iRegL src1, iRegL src2,
10970 immI src3) %{
10971 match(Set dst (OrL src1 (URShiftL src2 src3)));
10972
10973 ins_cost(1.9 * INSN_COST);
10974 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10975
10976 ins_encode %{
10977 __ orr(as_Register($dst$$reg),
10978 as_Register($src1$$reg),
10979 as_Register($src2$$reg),
10980 Assembler::LSR,
10981 $src3$$constant & 0x3f);
10982 %}
10983
10984 ins_pipe(ialu_reg_reg_shift);
10985 %}
10986
10987 // This pattern is automatically generated from aarch64_ad.m4
10988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10989 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10990 iRegIorL2I src1, iRegIorL2I src2,
10991 immI src3) %{
10992 match(Set dst (OrI src1 (RShiftI src2 src3)));
10993
10994 ins_cost(1.9 * INSN_COST);
10995 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10996
10997 ins_encode %{
10998 __ orrw(as_Register($dst$$reg),
10999 as_Register($src1$$reg),
11000 as_Register($src2$$reg),
11001 Assembler::ASR,
11002 $src3$$constant & 0x1f);
11003 %}
11004
11005 ins_pipe(ialu_reg_reg_shift);
11006 %}
11007
11008 // This pattern is automatically generated from aarch64_ad.m4
11009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11010 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11011 iRegL src1, iRegL src2,
11012 immI src3) %{
11013 match(Set dst (OrL src1 (RShiftL src2 src3)));
11014
11015 ins_cost(1.9 * INSN_COST);
11016 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11017
11018 ins_encode %{
11019 __ orr(as_Register($dst$$reg),
11020 as_Register($src1$$reg),
11021 as_Register($src2$$reg),
11022 Assembler::ASR,
11023 $src3$$constant & 0x3f);
11024 %}
11025
11026 ins_pipe(ialu_reg_reg_shift);
11027 %}
11028
11029 // This pattern is automatically generated from aarch64_ad.m4
11030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11031 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11032 iRegIorL2I src1, iRegIorL2I src2,
11033 immI src3) %{
11034 match(Set dst (OrI src1 (LShiftI src2 src3)));
11035
11036 ins_cost(1.9 * INSN_COST);
11037 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11038
11039 ins_encode %{
11040 __ orrw(as_Register($dst$$reg),
11041 as_Register($src1$$reg),
11042 as_Register($src2$$reg),
11043 Assembler::LSL,
11044 $src3$$constant & 0x1f);
11045 %}
11046
11047 ins_pipe(ialu_reg_reg_shift);
11048 %}
11049
11050 // This pattern is automatically generated from aarch64_ad.m4
11051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11052 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11053 iRegL src1, iRegL src2,
11054 immI src3) %{
11055 match(Set dst (OrL src1 (LShiftL src2 src3)));
11056
11057 ins_cost(1.9 * INSN_COST);
11058 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11059
11060 ins_encode %{
11061 __ orr(as_Register($dst$$reg),
11062 as_Register($src1$$reg),
11063 as_Register($src2$$reg),
11064 Assembler::LSL,
11065 $src3$$constant & 0x3f);
11066 %}
11067
11068 ins_pipe(ialu_reg_reg_shift);
11069 %}
11070
11071 // This pattern is automatically generated from aarch64_ad.m4
11072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11073 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11074 iRegIorL2I src1, iRegIorL2I src2,
11075 immI src3) %{
11076 match(Set dst (OrI src1 (RotateRight src2 src3)));
11077
11078 ins_cost(1.9 * INSN_COST);
11079 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11080
11081 ins_encode %{
11082 __ orrw(as_Register($dst$$reg),
11083 as_Register($src1$$reg),
11084 as_Register($src2$$reg),
11085 Assembler::ROR,
11086 $src3$$constant & 0x1f);
11087 %}
11088
11089 ins_pipe(ialu_reg_reg_shift);
11090 %}
11091
11092 // This pattern is automatically generated from aarch64_ad.m4
11093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11094 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11095 iRegL src1, iRegL src2,
11096 immI src3) %{
11097 match(Set dst (OrL src1 (RotateRight src2 src3)));
11098
11099 ins_cost(1.9 * INSN_COST);
11100 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11101
11102 ins_encode %{
11103 __ orr(as_Register($dst$$reg),
11104 as_Register($src1$$reg),
11105 as_Register($src2$$reg),
11106 Assembler::ROR,
11107 $src3$$constant & 0x3f);
11108 %}
11109
11110 ins_pipe(ialu_reg_reg_shift);
11111 %}
11112
11113 // This pattern is automatically generated from aarch64_ad.m4
11114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11115 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11116 iRegIorL2I src1, iRegIorL2I src2,
11117 immI src3) %{
11118 match(Set dst (AddI src1 (URShiftI src2 src3)));
11119
11120 ins_cost(1.9 * INSN_COST);
11121 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11122
11123 ins_encode %{
11124 __ addw(as_Register($dst$$reg),
11125 as_Register($src1$$reg),
11126 as_Register($src2$$reg),
11127 Assembler::LSR,
11128 $src3$$constant & 0x1f);
11129 %}
11130
11131 ins_pipe(ialu_reg_reg_shift);
11132 %}
11133
11134 // This pattern is automatically generated from aarch64_ad.m4
11135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11136 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11137 iRegL src1, iRegL src2,
11138 immI src3) %{
11139 match(Set dst (AddL src1 (URShiftL src2 src3)));
11140
11141 ins_cost(1.9 * INSN_COST);
11142 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11143
11144 ins_encode %{
11145 __ add(as_Register($dst$$reg),
11146 as_Register($src1$$reg),
11147 as_Register($src2$$reg),
11148 Assembler::LSR,
11149 $src3$$constant & 0x3f);
11150 %}
11151
11152 ins_pipe(ialu_reg_reg_shift);
11153 %}
11154
11155 // This pattern is automatically generated from aarch64_ad.m4
11156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11157 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11158 iRegIorL2I src1, iRegIorL2I src2,
11159 immI src3) %{
11160 match(Set dst (AddI src1 (RShiftI src2 src3)));
11161
11162 ins_cost(1.9 * INSN_COST);
11163 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11164
11165 ins_encode %{
11166 __ addw(as_Register($dst$$reg),
11167 as_Register($src1$$reg),
11168 as_Register($src2$$reg),
11169 Assembler::ASR,
11170 $src3$$constant & 0x1f);
11171 %}
11172
11173 ins_pipe(ialu_reg_reg_shift);
11174 %}
11175
11176 // This pattern is automatically generated from aarch64_ad.m4
11177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11178 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11179 iRegL src1, iRegL src2,
11180 immI src3) %{
11181 match(Set dst (AddL src1 (RShiftL src2 src3)));
11182
11183 ins_cost(1.9 * INSN_COST);
11184 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11185
11186 ins_encode %{
11187 __ add(as_Register($dst$$reg),
11188 as_Register($src1$$reg),
11189 as_Register($src2$$reg),
11190 Assembler::ASR,
11191 $src3$$constant & 0x3f);
11192 %}
11193
11194 ins_pipe(ialu_reg_reg_shift);
11195 %}
11196
11197 // This pattern is automatically generated from aarch64_ad.m4
11198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11199 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11200 iRegIorL2I src1, iRegIorL2I src2,
11201 immI src3) %{
11202 match(Set dst (AddI src1 (LShiftI src2 src3)));
11203
11204 ins_cost(1.9 * INSN_COST);
11205 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11206
11207 ins_encode %{
11208 __ addw(as_Register($dst$$reg),
11209 as_Register($src1$$reg),
11210 as_Register($src2$$reg),
11211 Assembler::LSL,
11212 $src3$$constant & 0x1f);
11213 %}
11214
11215 ins_pipe(ialu_reg_reg_shift);
11216 %}
11217
11218 // This pattern is automatically generated from aarch64_ad.m4
11219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11220 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11221 iRegL src1, iRegL src2,
11222 immI src3) %{
11223 match(Set dst (AddL src1 (LShiftL src2 src3)));
11224
11225 ins_cost(1.9 * INSN_COST);
11226 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11227
11228 ins_encode %{
11229 __ add(as_Register($dst$$reg),
11230 as_Register($src1$$reg),
11231 as_Register($src2$$reg),
11232 Assembler::LSL,
11233 $src3$$constant & 0x3f);
11234 %}
11235
11236 ins_pipe(ialu_reg_reg_shift);
11237 %}
11238
11239 // This pattern is automatically generated from aarch64_ad.m4
11240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11241 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11242 iRegIorL2I src1, iRegIorL2I src2,
11243 immI src3) %{
11244 match(Set dst (SubI src1 (URShiftI src2 src3)));
11245
11246 ins_cost(1.9 * INSN_COST);
11247 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11248
11249 ins_encode %{
11250 __ subw(as_Register($dst$$reg),
11251 as_Register($src1$$reg),
11252 as_Register($src2$$reg),
11253 Assembler::LSR,
11254 $src3$$constant & 0x1f);
11255 %}
11256
11257 ins_pipe(ialu_reg_reg_shift);
11258 %}
11259
11260 // This pattern is automatically generated from aarch64_ad.m4
11261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11262 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11263 iRegL src1, iRegL src2,
11264 immI src3) %{
11265 match(Set dst (SubL src1 (URShiftL src2 src3)));
11266
11267 ins_cost(1.9 * INSN_COST);
11268 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11269
11270 ins_encode %{
11271 __ sub(as_Register($dst$$reg),
11272 as_Register($src1$$reg),
11273 as_Register($src2$$reg),
11274 Assembler::LSR,
11275 $src3$$constant & 0x3f);
11276 %}
11277
11278 ins_pipe(ialu_reg_reg_shift);
11279 %}
11280
11281 // This pattern is automatically generated from aarch64_ad.m4
11282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11283 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11284 iRegIorL2I src1, iRegIorL2I src2,
11285 immI src3) %{
11286 match(Set dst (SubI src1 (RShiftI src2 src3)));
11287
11288 ins_cost(1.9 * INSN_COST);
11289 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11290
11291 ins_encode %{
11292 __ subw(as_Register($dst$$reg),
11293 as_Register($src1$$reg),
11294 as_Register($src2$$reg),
11295 Assembler::ASR,
11296 $src3$$constant & 0x1f);
11297 %}
11298
11299 ins_pipe(ialu_reg_reg_shift);
11300 %}
11301
11302 // This pattern is automatically generated from aarch64_ad.m4
11303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11304 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11305 iRegL src1, iRegL src2,
11306 immI src3) %{
11307 match(Set dst (SubL src1 (RShiftL src2 src3)));
11308
11309 ins_cost(1.9 * INSN_COST);
11310 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11311
11312 ins_encode %{
11313 __ sub(as_Register($dst$$reg),
11314 as_Register($src1$$reg),
11315 as_Register($src2$$reg),
11316 Assembler::ASR,
11317 $src3$$constant & 0x3f);
11318 %}
11319
11320 ins_pipe(ialu_reg_reg_shift);
11321 %}
11322
11323 // This pattern is automatically generated from aarch64_ad.m4
11324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11325 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11326 iRegIorL2I src1, iRegIorL2I src2,
11327 immI src3) %{
11328 match(Set dst (SubI src1 (LShiftI src2 src3)));
11329
11330 ins_cost(1.9 * INSN_COST);
11331 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11332
11333 ins_encode %{
11334 __ subw(as_Register($dst$$reg),
11335 as_Register($src1$$reg),
11336 as_Register($src2$$reg),
11337 Assembler::LSL,
11338 $src3$$constant & 0x1f);
11339 %}
11340
11341 ins_pipe(ialu_reg_reg_shift);
11342 %}
11343
11344 // This pattern is automatically generated from aarch64_ad.m4
11345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11346 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11347 iRegL src1, iRegL src2,
11348 immI src3) %{
11349 match(Set dst (SubL src1 (LShiftL src2 src3)));
11350
11351 ins_cost(1.9 * INSN_COST);
11352 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11353
11354 ins_encode %{
11355 __ sub(as_Register($dst$$reg),
11356 as_Register($src1$$reg),
11357 as_Register($src2$$reg),
11358 Assembler::LSL,
11359 $src3$$constant & 0x3f);
11360 %}
11361
11362 ins_pipe(ialu_reg_reg_shift);
11363 %}
11364
11365 // This pattern is automatically generated from aarch64_ad.m4
11366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11367
11368 // Shift Left followed by Shift Right.
11369 // This idiom is used by the compiler for the i2b bytecode etc.
11370 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11371 %{
11372 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11373 ins_cost(INSN_COST * 2);
11374 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11375 ins_encode %{
11376 int lshift = $lshift_count$$constant & 63;
11377 int rshift = $rshift_count$$constant & 63;
11378 int s = 63 - lshift;
11379 int r = (rshift - lshift) & 63;
11380 __ sbfm(as_Register($dst$$reg),
11381 as_Register($src$$reg),
11382 r, s);
11383 %}
11384
11385 ins_pipe(ialu_reg_shift);
11386 %}
11387
11388 // This pattern is automatically generated from aarch64_ad.m4
11389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11390
11391 // Shift Left followed by Shift Right.
11392 // This idiom is used by the compiler for the i2b bytecode etc.
11393 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11394 %{
11395 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11396 ins_cost(INSN_COST * 2);
11397 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11398 ins_encode %{
11399 int lshift = $lshift_count$$constant & 31;
11400 int rshift = $rshift_count$$constant & 31;
11401 int s = 31 - lshift;
11402 int r = (rshift - lshift) & 31;
11403 __ sbfmw(as_Register($dst$$reg),
11404 as_Register($src$$reg),
11405 r, s);
11406 %}
11407
11408 ins_pipe(ialu_reg_shift);
11409 %}
11410
11411 // This pattern is automatically generated from aarch64_ad.m4
11412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11413
11414 // Shift Left followed by Shift Right.
11415 // This idiom is used by the compiler for the i2b bytecode etc.
11416 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11417 %{
11418 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11419 ins_cost(INSN_COST * 2);
11420 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11421 ins_encode %{
11422 int lshift = $lshift_count$$constant & 63;
11423 int rshift = $rshift_count$$constant & 63;
11424 int s = 63 - lshift;
11425 int r = (rshift - lshift) & 63;
11426 __ ubfm(as_Register($dst$$reg),
11427 as_Register($src$$reg),
11428 r, s);
11429 %}
11430
11431 ins_pipe(ialu_reg_shift);
11432 %}
11433
11434 // This pattern is automatically generated from aarch64_ad.m4
11435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11436
11437 // Shift Left followed by Shift Right.
11438 // This idiom is used by the compiler for the i2b bytecode etc.
11439 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11440 %{
11441 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11442 ins_cost(INSN_COST * 2);
11443 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11444 ins_encode %{
11445 int lshift = $lshift_count$$constant & 31;
11446 int rshift = $rshift_count$$constant & 31;
11447 int s = 31 - lshift;
11448 int r = (rshift - lshift) & 31;
11449 __ ubfmw(as_Register($dst$$reg),
11450 as_Register($src$$reg),
11451 r, s);
11452 %}
11453
11454 ins_pipe(ialu_reg_shift);
11455 %}
11456
11457 // Bitfield extract with shift & mask
11458
11459 // This pattern is automatically generated from aarch64_ad.m4
11460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11461 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11462 %{
11463 match(Set dst (AndI (URShiftI src rshift) mask));
11464 // Make sure we are not going to exceed what ubfxw can do.
11465 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11466
11467 ins_cost(INSN_COST);
11468 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11469 ins_encode %{
11470 int rshift = $rshift$$constant & 31;
11471 intptr_t mask = $mask$$constant;
11472 int width = exact_log2(mask+1);
11473 __ ubfxw(as_Register($dst$$reg),
11474 as_Register($src$$reg), rshift, width);
11475 %}
11476 ins_pipe(ialu_reg_shift);
11477 %}
11478
11479 // This pattern is automatically generated from aarch64_ad.m4
11480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11481 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11482 %{
11483 match(Set dst (AndL (URShiftL src rshift) mask));
11484 // Make sure we are not going to exceed what ubfx can do.
11485 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11486
11487 ins_cost(INSN_COST);
11488 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11489 ins_encode %{
11490 int rshift = $rshift$$constant & 63;
11491 intptr_t mask = $mask$$constant;
11492 int width = exact_log2_long(mask+1);
11493 __ ubfx(as_Register($dst$$reg),
11494 as_Register($src$$reg), rshift, width);
11495 %}
11496 ins_pipe(ialu_reg_shift);
11497 %}
11498
11499
11500 // This pattern is automatically generated from aarch64_ad.m4
11501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11502
11503 // We can use ubfx when extending an And with a mask when we know mask
11504 // is positive. We know that because immI_bitmask guarantees it.
11505 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11506 %{
11507 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11508 // Make sure we are not going to exceed what ubfxw can do.
11509 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11510
11511 ins_cost(INSN_COST * 2);
11512 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11513 ins_encode %{
11514 int rshift = $rshift$$constant & 31;
11515 intptr_t mask = $mask$$constant;
11516 int width = exact_log2(mask+1);
11517 __ ubfx(as_Register($dst$$reg),
11518 as_Register($src$$reg), rshift, width);
11519 %}
11520 ins_pipe(ialu_reg_shift);
11521 %}
11522
11523
11524 // This pattern is automatically generated from aarch64_ad.m4
11525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11526
11527 // We can use ubfiz when masking by a positive number and then left shifting the result.
11528 // We know that the mask is positive because immI_bitmask guarantees it.
11529 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11530 %{
11531 match(Set dst (LShiftI (AndI src mask) lshift));
11532 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11533
11534 ins_cost(INSN_COST);
11535 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11536 ins_encode %{
11537 int lshift = $lshift$$constant & 31;
11538 intptr_t mask = $mask$$constant;
11539 int width = exact_log2(mask+1);
11540 __ ubfizw(as_Register($dst$$reg),
11541 as_Register($src$$reg), lshift, width);
11542 %}
11543 ins_pipe(ialu_reg_shift);
11544 %}
11545
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548
11549 // We can use ubfiz when masking by a positive number and then left shifting the result.
11550 // We know that the mask is positive because immL_bitmask guarantees it.
11551 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11552 %{
11553 match(Set dst (LShiftL (AndL src mask) lshift));
11554 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11555
11556 ins_cost(INSN_COST);
11557 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11558 ins_encode %{
11559 int lshift = $lshift$$constant & 63;
11560 intptr_t mask = $mask$$constant;
11561 int width = exact_log2_long(mask+1);
11562 __ ubfiz(as_Register($dst$$reg),
11563 as_Register($src$$reg), lshift, width);
11564 %}
11565 ins_pipe(ialu_reg_shift);
11566 %}
11567
11568 // This pattern is automatically generated from aarch64_ad.m4
11569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11570
11571 // We can use ubfiz when masking by a positive number and then left shifting the result.
11572 // We know that the mask is positive because immI_bitmask guarantees it.
11573 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11574 %{
11575 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11576 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11577
11578 ins_cost(INSN_COST);
11579 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11580 ins_encode %{
11581 int lshift = $lshift$$constant & 31;
11582 intptr_t mask = $mask$$constant;
11583 int width = exact_log2(mask+1);
11584 __ ubfizw(as_Register($dst$$reg),
11585 as_Register($src$$reg), lshift, width);
11586 %}
11587 ins_pipe(ialu_reg_shift);
11588 %}
11589
11590 // This pattern is automatically generated from aarch64_ad.m4
11591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11592
11593 // We can use ubfiz when masking by a positive number and then left shifting the result.
11594 // We know that the mask is positive because immL_bitmask guarantees it.
11595 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11596 %{
11597 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11598 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11599
11600 ins_cost(INSN_COST);
11601 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11602 ins_encode %{
11603 int lshift = $lshift$$constant & 63;
11604 intptr_t mask = $mask$$constant;
11605 int width = exact_log2_long(mask+1);
11606 __ ubfiz(as_Register($dst$$reg),
11607 as_Register($src$$reg), lshift, width);
11608 %}
11609 ins_pipe(ialu_reg_shift);
11610 %}
11611
11612
11613 // This pattern is automatically generated from aarch64_ad.m4
11614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11615
11616 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11617 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11618 %{
11619 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11620 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11621
11622 ins_cost(INSN_COST);
11623 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11624 ins_encode %{
11625 int lshift = $lshift$$constant & 63;
11626 intptr_t mask = $mask$$constant;
11627 int width = exact_log2(mask+1);
11628 __ ubfiz(as_Register($dst$$reg),
11629 as_Register($src$$reg), lshift, width);
11630 %}
11631 ins_pipe(ialu_reg_shift);
11632 %}
11633
11634 // This pattern is automatically generated from aarch64_ad.m4
11635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11636
11637 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11638 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11639 %{
11640 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11641 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11642
11643 ins_cost(INSN_COST);
11644 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11645 ins_encode %{
11646 int lshift = $lshift$$constant & 31;
11647 intptr_t mask = $mask$$constant;
11648 int width = exact_log2(mask+1);
11649 __ ubfiz(as_Register($dst$$reg),
11650 as_Register($src$$reg), lshift, width);
11651 %}
11652 ins_pipe(ialu_reg_shift);
11653 %}
11654
11655 // This pattern is automatically generated from aarch64_ad.m4
11656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11657
11658 // Can skip int2long conversions after AND with small bitmask
11659 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11660 %{
11661 match(Set dst (ConvI2L (AndI src msk)));
11662 ins_cost(INSN_COST);
11663 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11664 ins_encode %{
11665 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11666 %}
11667 ins_pipe(ialu_reg_shift);
11668 %}
11669
11670
11671 // Rotations
11672
11673 // This pattern is automatically generated from aarch64_ad.m4
11674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11675 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11676 %{
11677 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11678 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11679
11680 ins_cost(INSN_COST);
11681 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11682
11683 ins_encode %{
11684 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11685 $rshift$$constant & 63);
11686 %}
11687 ins_pipe(ialu_reg_reg_extr);
11688 %}
11689
11690
11691 // This pattern is automatically generated from aarch64_ad.m4
11692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11693 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11694 %{
11695 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11696 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11697
11698 ins_cost(INSN_COST);
11699 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11700
11701 ins_encode %{
11702 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11703 $rshift$$constant & 31);
11704 %}
11705 ins_pipe(ialu_reg_reg_extr);
11706 %}
11707
11708
11709 // This pattern is automatically generated from aarch64_ad.m4
11710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11711 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11712 %{
11713 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11714 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11715
11716 ins_cost(INSN_COST);
11717 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11718
11719 ins_encode %{
11720 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11721 $rshift$$constant & 63);
11722 %}
11723 ins_pipe(ialu_reg_reg_extr);
11724 %}
11725
11726
11727 // This pattern is automatically generated from aarch64_ad.m4
11728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11729 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11730 %{
11731 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11732 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11733
11734 ins_cost(INSN_COST);
11735 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11736
11737 ins_encode %{
11738 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11739 $rshift$$constant & 31);
11740 %}
11741 ins_pipe(ialu_reg_reg_extr);
11742 %}
11743
11744 // This pattern is automatically generated from aarch64_ad.m4
11745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11746 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11747 %{
11748 match(Set dst (RotateRight src shift));
11749
11750 ins_cost(INSN_COST);
11751 format %{ "ror $dst, $src, $shift" %}
11752
11753 ins_encode %{
11754 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11755 $shift$$constant & 0x1f);
11756 %}
11757 ins_pipe(ialu_reg_reg_vshift);
11758 %}
11759
11760 // This pattern is automatically generated from aarch64_ad.m4
11761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11762 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11763 %{
11764 match(Set dst (RotateRight src shift));
11765
11766 ins_cost(INSN_COST);
11767 format %{ "ror $dst, $src, $shift" %}
11768
11769 ins_encode %{
11770 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11771 $shift$$constant & 0x3f);
11772 %}
11773 ins_pipe(ialu_reg_reg_vshift);
11774 %}
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11779 %{
11780 match(Set dst (RotateRight src shift));
11781
11782 ins_cost(INSN_COST);
11783 format %{ "ror $dst, $src, $shift" %}
11784
11785 ins_encode %{
11786 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11787 %}
11788 ins_pipe(ialu_reg_reg_vshift);
11789 %}
11790
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11794 %{
11795 match(Set dst (RotateRight src shift));
11796
11797 ins_cost(INSN_COST);
11798 format %{ "ror $dst, $src, $shift" %}
11799
11800 ins_encode %{
11801 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11802 %}
11803 ins_pipe(ialu_reg_reg_vshift);
11804 %}
11805
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11809 %{
11810 match(Set dst (RotateLeft src shift));
11811
11812 ins_cost(INSN_COST);
11813 format %{ "rol $dst, $src, $shift" %}
11814
11815 ins_encode %{
11816 __ subw(rscratch1, zr, as_Register($shift$$reg));
11817 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11818 %}
11819 ins_pipe(ialu_reg_reg_vshift);
11820 %}
11821
11822 // This pattern is automatically generated from aarch64_ad.m4
11823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11824 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11825 %{
11826 match(Set dst (RotateLeft src shift));
11827
11828 ins_cost(INSN_COST);
11829 format %{ "rol $dst, $src, $shift" %}
11830
11831 ins_encode %{
11832 __ subw(rscratch1, zr, as_Register($shift$$reg));
11833 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11834 %}
11835 ins_pipe(ialu_reg_reg_vshift);
11836 %}
11837
11838
11839 // Add/subtract (extended)
11840
11841 // This pattern is automatically generated from aarch64_ad.m4
11842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11843 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11844 %{
11845 match(Set dst (AddL src1 (ConvI2L src2)));
11846 ins_cost(INSN_COST);
11847 format %{ "add $dst, $src1, $src2, sxtw" %}
11848
11849 ins_encode %{
11850 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11851 as_Register($src2$$reg), ext::sxtw);
11852 %}
11853 ins_pipe(ialu_reg_reg);
11854 %}
11855
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11859 %{
11860 match(Set dst (SubL src1 (ConvI2L src2)));
11861 ins_cost(INSN_COST);
11862 format %{ "sub $dst, $src1, $src2, sxtw" %}
11863
11864 ins_encode %{
11865 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11866 as_Register($src2$$reg), ext::sxtw);
11867 %}
11868 ins_pipe(ialu_reg_reg);
11869 %}
11870
11871 // This pattern is automatically generated from aarch64_ad.m4
11872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11873 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11874 %{
11875 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11876 ins_cost(INSN_COST);
11877 format %{ "add $dst, $src1, $src2, sxth" %}
11878
11879 ins_encode %{
11880 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11881 as_Register($src2$$reg), ext::sxth);
11882 %}
11883 ins_pipe(ialu_reg_reg);
11884 %}
11885
11886 // This pattern is automatically generated from aarch64_ad.m4
11887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11888 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11889 %{
11890 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11891 ins_cost(INSN_COST);
11892 format %{ "add $dst, $src1, $src2, sxtb" %}
11893
11894 ins_encode %{
11895 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11896 as_Register($src2$$reg), ext::sxtb);
11897 %}
11898 ins_pipe(ialu_reg_reg);
11899 %}
11900
11901 // This pattern is automatically generated from aarch64_ad.m4
11902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11903 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11904 %{
11905 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11906 ins_cost(INSN_COST);
11907 format %{ "add $dst, $src1, $src2, uxtb" %}
11908
11909 ins_encode %{
11910 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11911 as_Register($src2$$reg), ext::uxtb);
11912 %}
11913 ins_pipe(ialu_reg_reg);
11914 %}
11915
11916 // This pattern is automatically generated from aarch64_ad.m4
11917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11918 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11919 %{
11920 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11921 ins_cost(INSN_COST);
11922 format %{ "add $dst, $src1, $src2, sxth" %}
11923
11924 ins_encode %{
11925 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11926 as_Register($src2$$reg), ext::sxth);
11927 %}
11928 ins_pipe(ialu_reg_reg);
11929 %}
11930
11931 // This pattern is automatically generated from aarch64_ad.m4
11932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11933 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11934 %{
11935 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11936 ins_cost(INSN_COST);
11937 format %{ "add $dst, $src1, $src2, sxtw" %}
11938
11939 ins_encode %{
11940 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11941 as_Register($src2$$reg), ext::sxtw);
11942 %}
11943 ins_pipe(ialu_reg_reg);
11944 %}
11945
11946 // This pattern is automatically generated from aarch64_ad.m4
11947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11948 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11949 %{
11950 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11951 ins_cost(INSN_COST);
11952 format %{ "add $dst, $src1, $src2, sxtb" %}
11953
11954 ins_encode %{
11955 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11956 as_Register($src2$$reg), ext::sxtb);
11957 %}
11958 ins_pipe(ialu_reg_reg);
11959 %}
11960
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11964 %{
11965 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11966 ins_cost(INSN_COST);
11967 format %{ "add $dst, $src1, $src2, uxtb" %}
11968
11969 ins_encode %{
11970 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11971 as_Register($src2$$reg), ext::uxtb);
11972 %}
11973 ins_pipe(ialu_reg_reg);
11974 %}
11975
11976 // This pattern is automatically generated from aarch64_ad.m4
11977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11978 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11979 %{
11980 match(Set dst (AddI src1 (AndI src2 mask)));
11981 ins_cost(INSN_COST);
11982 format %{ "addw $dst, $src1, $src2, uxtb" %}
11983
11984 ins_encode %{
11985 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11986 as_Register($src2$$reg), ext::uxtb);
11987 %}
11988 ins_pipe(ialu_reg_reg);
11989 %}
11990
11991 // This pattern is automatically generated from aarch64_ad.m4
11992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11993 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11994 %{
11995 match(Set dst (AddI src1 (AndI src2 mask)));
11996 ins_cost(INSN_COST);
11997 format %{ "addw $dst, $src1, $src2, uxth" %}
11998
11999 ins_encode %{
12000 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12001 as_Register($src2$$reg), ext::uxth);
12002 %}
12003 ins_pipe(ialu_reg_reg);
12004 %}
12005
12006 // This pattern is automatically generated from aarch64_ad.m4
12007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12008 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12009 %{
12010 match(Set dst (AddL src1 (AndL src2 mask)));
12011 ins_cost(INSN_COST);
12012 format %{ "add $dst, $src1, $src2, uxtb" %}
12013
12014 ins_encode %{
12015 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12016 as_Register($src2$$reg), ext::uxtb);
12017 %}
12018 ins_pipe(ialu_reg_reg);
12019 %}
12020
12021 // This pattern is automatically generated from aarch64_ad.m4
12022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12023 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12024 %{
12025 match(Set dst (AddL src1 (AndL src2 mask)));
12026 ins_cost(INSN_COST);
12027 format %{ "add $dst, $src1, $src2, uxth" %}
12028
12029 ins_encode %{
12030 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12031 as_Register($src2$$reg), ext::uxth);
12032 %}
12033 ins_pipe(ialu_reg_reg);
12034 %}
12035
12036 // This pattern is automatically generated from aarch64_ad.m4
12037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12038 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12039 %{
12040 match(Set dst (AddL src1 (AndL src2 mask)));
12041 ins_cost(INSN_COST);
12042 format %{ "add $dst, $src1, $src2, uxtw" %}
12043
12044 ins_encode %{
12045 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12046 as_Register($src2$$reg), ext::uxtw);
12047 %}
12048 ins_pipe(ialu_reg_reg);
12049 %}
12050
12051 // This pattern is automatically generated from aarch64_ad.m4
12052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12053 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12054 %{
12055 match(Set dst (SubI src1 (AndI src2 mask)));
12056 ins_cost(INSN_COST);
12057 format %{ "subw $dst, $src1, $src2, uxtb" %}
12058
12059 ins_encode %{
12060 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12061 as_Register($src2$$reg), ext::uxtb);
12062 %}
12063 ins_pipe(ialu_reg_reg);
12064 %}
12065
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12069 %{
12070 match(Set dst (SubI src1 (AndI src2 mask)));
12071 ins_cost(INSN_COST);
12072 format %{ "subw $dst, $src1, $src2, uxth" %}
12073
12074 ins_encode %{
12075 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12076 as_Register($src2$$reg), ext::uxth);
12077 %}
12078 ins_pipe(ialu_reg_reg);
12079 %}
12080
12081 // This pattern is automatically generated from aarch64_ad.m4
12082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12083 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12084 %{
12085 match(Set dst (SubL src1 (AndL src2 mask)));
12086 ins_cost(INSN_COST);
12087 format %{ "sub $dst, $src1, $src2, uxtb" %}
12088
12089 ins_encode %{
12090 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12091 as_Register($src2$$reg), ext::uxtb);
12092 %}
12093 ins_pipe(ialu_reg_reg);
12094 %}
12095
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12099 %{
12100 match(Set dst (SubL src1 (AndL src2 mask)));
12101 ins_cost(INSN_COST);
12102 format %{ "sub $dst, $src1, $src2, uxth" %}
12103
12104 ins_encode %{
12105 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12106 as_Register($src2$$reg), ext::uxth);
12107 %}
12108 ins_pipe(ialu_reg_reg);
12109 %}
12110
12111 // This pattern is automatically generated from aarch64_ad.m4
12112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12113 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12114 %{
12115 match(Set dst (SubL src1 (AndL src2 mask)));
12116 ins_cost(INSN_COST);
12117 format %{ "sub $dst, $src1, $src2, uxtw" %}
12118
12119 ins_encode %{
12120 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12121 as_Register($src2$$reg), ext::uxtw);
12122 %}
12123 ins_pipe(ialu_reg_reg);
12124 %}
12125
12126
12127 // This pattern is automatically generated from aarch64_ad.m4
12128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12129 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12130 %{
12131 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12132 ins_cost(1.9 * INSN_COST);
12133 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12134
12135 ins_encode %{
12136 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12137 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12138 %}
12139 ins_pipe(ialu_reg_reg_shift);
12140 %}
12141
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12145 %{
12146 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12147 ins_cost(1.9 * INSN_COST);
12148 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12149
12150 ins_encode %{
12151 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12152 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12153 %}
12154 ins_pipe(ialu_reg_reg_shift);
12155 %}
12156
12157 // This pattern is automatically generated from aarch64_ad.m4
12158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12159 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12160 %{
12161 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12162 ins_cost(1.9 * INSN_COST);
12163 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12164
12165 ins_encode %{
12166 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12167 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12168 %}
12169 ins_pipe(ialu_reg_reg_shift);
12170 %}
12171
12172 // This pattern is automatically generated from aarch64_ad.m4
12173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12174 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12175 %{
12176 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12177 ins_cost(1.9 * INSN_COST);
12178 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12179
12180 ins_encode %{
12181 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12182 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12183 %}
12184 ins_pipe(ialu_reg_reg_shift);
12185 %}
12186
12187 // This pattern is automatically generated from aarch64_ad.m4
12188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12189 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12190 %{
12191 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12192 ins_cost(1.9 * INSN_COST);
12193 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12194
12195 ins_encode %{
12196 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12197 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12198 %}
12199 ins_pipe(ialu_reg_reg_shift);
12200 %}
12201
12202 // This pattern is automatically generated from aarch64_ad.m4
12203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12204 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12205 %{
12206 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12207 ins_cost(1.9 * INSN_COST);
12208 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12209
12210 ins_encode %{
12211 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12212 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12213 %}
12214 ins_pipe(ialu_reg_reg_shift);
12215 %}
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12220 %{
12221 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12222 ins_cost(1.9 * INSN_COST);
12223 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12224
12225 ins_encode %{
12226 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12227 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12228 %}
12229 ins_pipe(ialu_reg_reg_shift);
12230 %}
12231
12232 // This pattern is automatically generated from aarch64_ad.m4
12233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12234 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12235 %{
12236 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12237 ins_cost(1.9 * INSN_COST);
12238 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12239
12240 ins_encode %{
12241 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12242 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12243 %}
12244 ins_pipe(ialu_reg_reg_shift);
12245 %}
12246
12247 // This pattern is automatically generated from aarch64_ad.m4
12248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12249 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12250 %{
12251 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12252 ins_cost(1.9 * INSN_COST);
12253 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12254
12255 ins_encode %{
12256 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12257 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12258 %}
12259 ins_pipe(ialu_reg_reg_shift);
12260 %}
12261
12262 // This pattern is automatically generated from aarch64_ad.m4
12263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12264 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12265 %{
12266 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12267 ins_cost(1.9 * INSN_COST);
12268 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12269
12270 ins_encode %{
12271 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12272 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12273 %}
12274 ins_pipe(ialu_reg_reg_shift);
12275 %}
12276
12277 // This pattern is automatically generated from aarch64_ad.m4
12278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12279 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12280 %{
12281 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12282 ins_cost(1.9 * INSN_COST);
12283 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12284
12285 ins_encode %{
12286 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12287 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12288 %}
12289 ins_pipe(ialu_reg_reg_shift);
12290 %}
12291
12292 // This pattern is automatically generated from aarch64_ad.m4
12293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12294 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12295 %{
12296 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12297 ins_cost(1.9 * INSN_COST);
12298 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12299
12300 ins_encode %{
12301 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12302 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12303 %}
12304 ins_pipe(ialu_reg_reg_shift);
12305 %}
12306
12307 // This pattern is automatically generated from aarch64_ad.m4
12308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12309 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12310 %{
12311 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12312 ins_cost(1.9 * INSN_COST);
12313 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12314
12315 ins_encode %{
12316 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12317 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12318 %}
12319 ins_pipe(ialu_reg_reg_shift);
12320 %}
12321
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12325 %{
12326 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12327 ins_cost(1.9 * INSN_COST);
12328 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12329
12330 ins_encode %{
12331 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12332 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12333 %}
12334 ins_pipe(ialu_reg_reg_shift);
12335 %}
12336
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12340 %{
12341 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12342 ins_cost(1.9 * INSN_COST);
12343 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12344
12345 ins_encode %{
12346 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12347 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12348 %}
12349 ins_pipe(ialu_reg_reg_shift);
12350 %}
12351
12352 // This pattern is automatically generated from aarch64_ad.m4
12353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12354 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12355 %{
12356 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12357 ins_cost(1.9 * INSN_COST);
12358 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12359
12360 ins_encode %{
12361 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12362 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12363 %}
12364 ins_pipe(ialu_reg_reg_shift);
12365 %}
12366
12367 // This pattern is automatically generated from aarch64_ad.m4
12368 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12369 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12370 %{
12371 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12372 ins_cost(1.9 * INSN_COST);
12373 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12374
12375 ins_encode %{
12376 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12377 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12378 %}
12379 ins_pipe(ialu_reg_reg_shift);
12380 %}
12381
12382 // This pattern is automatically generated from aarch64_ad.m4
12383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12384 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12385 %{
12386 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12387 ins_cost(1.9 * INSN_COST);
12388 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12389
12390 ins_encode %{
12391 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12392 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12393 %}
12394 ins_pipe(ialu_reg_reg_shift);
12395 %}
12396
12397 // This pattern is automatically generated from aarch64_ad.m4
12398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12399 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12400 %{
12401 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12402 ins_cost(1.9 * INSN_COST);
12403 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12404
12405 ins_encode %{
12406 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12407 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12408 %}
12409 ins_pipe(ialu_reg_reg_shift);
12410 %}
12411
12412 // This pattern is automatically generated from aarch64_ad.m4
12413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12414 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12415 %{
12416 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12417 ins_cost(1.9 * INSN_COST);
12418 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12419
12420 ins_encode %{
12421 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12422 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12423 %}
12424 ins_pipe(ialu_reg_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12430 %{
12431 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12432 ins_cost(1.9 * INSN_COST);
12433 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12434
12435 ins_encode %{
12436 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12437 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12438 %}
12439 ins_pipe(ialu_reg_reg_shift);
12440 %}
12441
12442 // This pattern is automatically generated from aarch64_ad.m4
12443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12444 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12445 %{
12446 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12447 ins_cost(1.9 * INSN_COST);
12448 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12449
12450 ins_encode %{
12451 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12452 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12453 %}
12454 ins_pipe(ialu_reg_reg_shift);
12455 %}
12456
12457 // This pattern is automatically generated from aarch64_ad.m4
12458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12459 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12460 %{
12461 effect(DEF dst, USE src1, USE src2, USE cr);
12462 ins_cost(INSN_COST * 2);
12463 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12464
12465 ins_encode %{
12466 __ cselw($dst$$Register,
12467 $src1$$Register,
12468 $src2$$Register,
12469 Assembler::LT);
12470 %}
12471 ins_pipe(icond_reg_reg);
12472 %}
12473
12474 // This pattern is automatically generated from aarch64_ad.m4
12475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12476 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12477 %{
12478 effect(DEF dst, USE src1, USE src2, USE cr);
12479 ins_cost(INSN_COST * 2);
12480 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12481
12482 ins_encode %{
12483 __ cselw($dst$$Register,
12484 $src1$$Register,
12485 $src2$$Register,
12486 Assembler::GT);
12487 %}
12488 ins_pipe(icond_reg_reg);
12489 %}
12490
12491 // This pattern is automatically generated from aarch64_ad.m4
12492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12493 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12494 %{
12495 effect(DEF dst, USE src1, USE cr);
12496 ins_cost(INSN_COST * 2);
12497 format %{ "cselw $dst, $src1, zr lt\t" %}
12498
12499 ins_encode %{
12500 __ cselw($dst$$Register,
12501 $src1$$Register,
12502 zr,
12503 Assembler::LT);
12504 %}
12505 ins_pipe(icond_reg);
12506 %}
12507
12508 // This pattern is automatically generated from aarch64_ad.m4
12509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12510 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12511 %{
12512 effect(DEF dst, USE src1, USE cr);
12513 ins_cost(INSN_COST * 2);
12514 format %{ "cselw $dst, $src1, zr gt\t" %}
12515
12516 ins_encode %{
12517 __ cselw($dst$$Register,
12518 $src1$$Register,
12519 zr,
12520 Assembler::GT);
12521 %}
12522 ins_pipe(icond_reg);
12523 %}
12524
12525 // This pattern is automatically generated from aarch64_ad.m4
12526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12527 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12528 %{
12529 effect(DEF dst, USE src1, USE cr);
12530 ins_cost(INSN_COST * 2);
12531 format %{ "csincw $dst, $src1, zr le\t" %}
12532
12533 ins_encode %{
12534 __ csincw($dst$$Register,
12535 $src1$$Register,
12536 zr,
12537 Assembler::LE);
12538 %}
12539 ins_pipe(icond_reg);
12540 %}
12541
12542 // This pattern is automatically generated from aarch64_ad.m4
12543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12544 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12545 %{
12546 effect(DEF dst, USE src1, USE cr);
12547 ins_cost(INSN_COST * 2);
12548 format %{ "csincw $dst, $src1, zr gt\t" %}
12549
12550 ins_encode %{
12551 __ csincw($dst$$Register,
12552 $src1$$Register,
12553 zr,
12554 Assembler::GT);
12555 %}
12556 ins_pipe(icond_reg);
12557 %}
12558
12559 // This pattern is automatically generated from aarch64_ad.m4
12560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12561 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12562 %{
12563 effect(DEF dst, USE src1, USE cr);
12564 ins_cost(INSN_COST * 2);
12565 format %{ "csinvw $dst, $src1, zr lt\t" %}
12566
12567 ins_encode %{
12568 __ csinvw($dst$$Register,
12569 $src1$$Register,
12570 zr,
12571 Assembler::LT);
12572 %}
12573 ins_pipe(icond_reg);
12574 %}
12575
12576 // This pattern is automatically generated from aarch64_ad.m4
12577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12578 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12579 %{
12580 effect(DEF dst, USE src1, USE cr);
12581 ins_cost(INSN_COST * 2);
12582 format %{ "csinvw $dst, $src1, zr ge\t" %}
12583
12584 ins_encode %{
12585 __ csinvw($dst$$Register,
12586 $src1$$Register,
12587 zr,
12588 Assembler::GE);
12589 %}
12590 ins_pipe(icond_reg);
12591 %}
12592
12593 // This pattern is automatically generated from aarch64_ad.m4
12594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12595 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12596 %{
12597 match(Set dst (MinI src imm));
12598 ins_cost(INSN_COST * 3);
12599 expand %{
12600 rFlagsReg cr;
12601 compI_reg_imm0(cr, src);
12602 cmovI_reg_imm0_lt(dst, src, cr);
12603 %}
12604 %}
12605
12606 // This pattern is automatically generated from aarch64_ad.m4
12607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12608 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12609 %{
12610 match(Set dst (MinI imm src));
12611 ins_cost(INSN_COST * 3);
12612 expand %{
12613 rFlagsReg cr;
12614 compI_reg_imm0(cr, src);
12615 cmovI_reg_imm0_lt(dst, src, cr);
12616 %}
12617 %}
12618
12619 // This pattern is automatically generated from aarch64_ad.m4
12620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12621 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12622 %{
12623 match(Set dst (MinI src imm));
12624 ins_cost(INSN_COST * 3);
12625 expand %{
12626 rFlagsReg cr;
12627 compI_reg_imm0(cr, src);
12628 cmovI_reg_imm1_le(dst, src, cr);
12629 %}
12630 %}
12631
12632 // This pattern is automatically generated from aarch64_ad.m4
12633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12634 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12635 %{
12636 match(Set dst (MinI imm src));
12637 ins_cost(INSN_COST * 3);
12638 expand %{
12639 rFlagsReg cr;
12640 compI_reg_imm0(cr, src);
12641 cmovI_reg_imm1_le(dst, src, cr);
12642 %}
12643 %}
12644
12645 // This pattern is automatically generated from aarch64_ad.m4
12646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12647 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12648 %{
12649 match(Set dst (MinI src imm));
12650 ins_cost(INSN_COST * 3);
12651 expand %{
12652 rFlagsReg cr;
12653 compI_reg_imm0(cr, src);
12654 cmovI_reg_immM1_lt(dst, src, cr);
12655 %}
12656 %}
12657
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12661 %{
12662 match(Set dst (MinI imm src));
12663 ins_cost(INSN_COST * 3);
12664 expand %{
12665 rFlagsReg cr;
12666 compI_reg_imm0(cr, src);
12667 cmovI_reg_immM1_lt(dst, src, cr);
12668 %}
12669 %}
12670
12671 // This pattern is automatically generated from aarch64_ad.m4
12672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12673 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12674 %{
12675 match(Set dst (MaxI src imm));
12676 ins_cost(INSN_COST * 3);
12677 expand %{
12678 rFlagsReg cr;
12679 compI_reg_imm0(cr, src);
12680 cmovI_reg_imm0_gt(dst, src, cr);
12681 %}
12682 %}
12683
12684 // This pattern is automatically generated from aarch64_ad.m4
12685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12686 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12687 %{
12688 match(Set dst (MaxI imm src));
12689 ins_cost(INSN_COST * 3);
12690 expand %{
12691 rFlagsReg cr;
12692 compI_reg_imm0(cr, src);
12693 cmovI_reg_imm0_gt(dst, src, cr);
12694 %}
12695 %}
12696
12697 // This pattern is automatically generated from aarch64_ad.m4
12698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12699 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12700 %{
12701 match(Set dst (MaxI src imm));
12702 ins_cost(INSN_COST * 3);
12703 expand %{
12704 rFlagsReg cr;
12705 compI_reg_imm0(cr, src);
12706 cmovI_reg_imm1_gt(dst, src, cr);
12707 %}
12708 %}
12709
12710 // This pattern is automatically generated from aarch64_ad.m4
12711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12712 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12713 %{
12714 match(Set dst (MaxI imm src));
12715 ins_cost(INSN_COST * 3);
12716 expand %{
12717 rFlagsReg cr;
12718 compI_reg_imm0(cr, src);
12719 cmovI_reg_imm1_gt(dst, src, cr);
12720 %}
12721 %}
12722
12723 // This pattern is automatically generated from aarch64_ad.m4
12724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12725 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12726 %{
12727 match(Set dst (MaxI src imm));
12728 ins_cost(INSN_COST * 3);
12729 expand %{
12730 rFlagsReg cr;
12731 compI_reg_imm0(cr, src);
12732 cmovI_reg_immM1_ge(dst, src, cr);
12733 %}
12734 %}
12735
12736 // This pattern is automatically generated from aarch64_ad.m4
12737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12738 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12739 %{
12740 match(Set dst (MaxI imm src));
12741 ins_cost(INSN_COST * 3);
12742 expand %{
12743 rFlagsReg cr;
12744 compI_reg_imm0(cr, src);
12745 cmovI_reg_immM1_ge(dst, src, cr);
12746 %}
12747 %}
12748
12749 // This pattern is automatically generated from aarch64_ad.m4
12750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12751 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12752 %{
12753 match(Set dst (ReverseI src));
12754 ins_cost(INSN_COST);
12755 format %{ "rbitw $dst, $src" %}
12756 ins_encode %{
12757 __ rbitw($dst$$Register, $src$$Register);
12758 %}
12759 ins_pipe(ialu_reg);
12760 %}
12761
12762 // This pattern is automatically generated from aarch64_ad.m4
12763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12764 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12765 %{
12766 match(Set dst (ReverseL src));
12767 ins_cost(INSN_COST);
12768 format %{ "rbit $dst, $src" %}
12769 ins_encode %{
12770 __ rbit($dst$$Register, $src$$Register);
12771 %}
12772 ins_pipe(ialu_reg);
12773 %}
12774
12775
12776 // END This section of the file is automatically generated. Do not edit --------------
12777
12778
12779 // ============================================================================
12780 // Floating Point Arithmetic Instructions
12781
12782 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12783 match(Set dst (AddHF src1 src2));
12784 format %{ "faddh $dst, $src1, $src2" %}
12785 ins_encode %{
12786 __ faddh($dst$$FloatRegister,
12787 $src1$$FloatRegister,
12788 $src2$$FloatRegister);
12789 %}
12790 ins_pipe(fp_dop_reg_reg_s);
12791 %}
12792
12793 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12794 match(Set dst (AddF src1 src2));
12795
12796 ins_cost(INSN_COST * 5);
12797 format %{ "fadds $dst, $src1, $src2" %}
12798
12799 ins_encode %{
12800 __ fadds(as_FloatRegister($dst$$reg),
12801 as_FloatRegister($src1$$reg),
12802 as_FloatRegister($src2$$reg));
12803 %}
12804
12805 ins_pipe(fp_dop_reg_reg_s);
12806 %}
12807
12808 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12809 match(Set dst (AddD src1 src2));
12810
12811 ins_cost(INSN_COST * 5);
12812 format %{ "faddd $dst, $src1, $src2" %}
12813
12814 ins_encode %{
12815 __ faddd(as_FloatRegister($dst$$reg),
12816 as_FloatRegister($src1$$reg),
12817 as_FloatRegister($src2$$reg));
12818 %}
12819
12820 ins_pipe(fp_dop_reg_reg_d);
12821 %}
12822
12823 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12824 match(Set dst (SubHF src1 src2));
12825 format %{ "fsubh $dst, $src1, $src2" %}
12826 ins_encode %{
12827 __ fsubh($dst$$FloatRegister,
12828 $src1$$FloatRegister,
12829 $src2$$FloatRegister);
12830 %}
12831 ins_pipe(fp_dop_reg_reg_s);
12832 %}
12833
12834 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12835 match(Set dst (SubF src1 src2));
12836
12837 ins_cost(INSN_COST * 5);
12838 format %{ "fsubs $dst, $src1, $src2" %}
12839
12840 ins_encode %{
12841 __ fsubs(as_FloatRegister($dst$$reg),
12842 as_FloatRegister($src1$$reg),
12843 as_FloatRegister($src2$$reg));
12844 %}
12845
12846 ins_pipe(fp_dop_reg_reg_s);
12847 %}
12848
12849 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12850 match(Set dst (SubD src1 src2));
12851
12852 ins_cost(INSN_COST * 5);
12853 format %{ "fsubd $dst, $src1, $src2" %}
12854
12855 ins_encode %{
12856 __ fsubd(as_FloatRegister($dst$$reg),
12857 as_FloatRegister($src1$$reg),
12858 as_FloatRegister($src2$$reg));
12859 %}
12860
12861 ins_pipe(fp_dop_reg_reg_d);
12862 %}
12863
12864 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12865 match(Set dst (MulHF src1 src2));
12866 format %{ "fmulh $dst, $src1, $src2" %}
12867 ins_encode %{
12868 __ fmulh($dst$$FloatRegister,
12869 $src1$$FloatRegister,
12870 $src2$$FloatRegister);
12871 %}
12872 ins_pipe(fp_dop_reg_reg_s);
12873 %}
12874
12875 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12876 match(Set dst (MulF src1 src2));
12877
12878 ins_cost(INSN_COST * 6);
12879 format %{ "fmuls $dst, $src1, $src2" %}
12880
12881 ins_encode %{
12882 __ fmuls(as_FloatRegister($dst$$reg),
12883 as_FloatRegister($src1$$reg),
12884 as_FloatRegister($src2$$reg));
12885 %}
12886
12887 ins_pipe(fp_dop_reg_reg_s);
12888 %}
12889
12890 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12891 match(Set dst (MulD src1 src2));
12892
12893 ins_cost(INSN_COST * 6);
12894 format %{ "fmuld $dst, $src1, $src2" %}
12895
12896 ins_encode %{
12897 __ fmuld(as_FloatRegister($dst$$reg),
12898 as_FloatRegister($src1$$reg),
12899 as_FloatRegister($src2$$reg));
12900 %}
12901
12902 ins_pipe(fp_dop_reg_reg_d);
12903 %}
12904
12905 // src1 * src2 + src3 (half-precision float)
12906 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12907 match(Set dst (FmaHF src3 (Binary src1 src2)));
12908 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12909 ins_encode %{
12910 assert(UseFMA, "Needs FMA instructions support.");
12911 __ fmaddh($dst$$FloatRegister,
12912 $src1$$FloatRegister,
12913 $src2$$FloatRegister,
12914 $src3$$FloatRegister);
12915 %}
12916 ins_pipe(pipe_class_default);
12917 %}
12918
12919 // src1 * src2 + src3
12920 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12921 match(Set dst (FmaF src3 (Binary src1 src2)));
12922
12923 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12924
12925 ins_encode %{
12926 assert(UseFMA, "Needs FMA instructions support.");
12927 __ fmadds(as_FloatRegister($dst$$reg),
12928 as_FloatRegister($src1$$reg),
12929 as_FloatRegister($src2$$reg),
12930 as_FloatRegister($src3$$reg));
12931 %}
12932
12933 ins_pipe(pipe_class_default);
12934 %}
12935
12936 // src1 * src2 + src3
12937 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12938 match(Set dst (FmaD src3 (Binary src1 src2)));
12939
12940 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12941
12942 ins_encode %{
12943 assert(UseFMA, "Needs FMA instructions support.");
12944 __ fmaddd(as_FloatRegister($dst$$reg),
12945 as_FloatRegister($src1$$reg),
12946 as_FloatRegister($src2$$reg),
12947 as_FloatRegister($src3$$reg));
12948 %}
12949
12950 ins_pipe(pipe_class_default);
12951 %}
12952
12953 // src1 * (-src2) + src3
12954 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12955 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12956 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12957
12958 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12959
12960 ins_encode %{
12961 assert(UseFMA, "Needs FMA instructions support.");
12962 __ fmsubs(as_FloatRegister($dst$$reg),
12963 as_FloatRegister($src1$$reg),
12964 as_FloatRegister($src2$$reg),
12965 as_FloatRegister($src3$$reg));
12966 %}
12967
12968 ins_pipe(pipe_class_default);
12969 %}
12970
12971 // src1 * (-src2) + src3
12972 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12973 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12974 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12975
12976 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12977
12978 ins_encode %{
12979 assert(UseFMA, "Needs FMA instructions support.");
12980 __ fmsubd(as_FloatRegister($dst$$reg),
12981 as_FloatRegister($src1$$reg),
12982 as_FloatRegister($src2$$reg),
12983 as_FloatRegister($src3$$reg));
12984 %}
12985
12986 ins_pipe(pipe_class_default);
12987 %}
12988
12989 // src1 * (-src2) - src3
12990 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12991 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12992 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12993
12994 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
12995
12996 ins_encode %{
12997 assert(UseFMA, "Needs FMA instructions support.");
12998 __ fnmadds(as_FloatRegister($dst$$reg),
12999 as_FloatRegister($src1$$reg),
13000 as_FloatRegister($src2$$reg),
13001 as_FloatRegister($src3$$reg));
13002 %}
13003
13004 ins_pipe(pipe_class_default);
13005 %}
13006
13007 // src1 * (-src2) - src3
13008 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13009 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13010 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13011
13012 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13013
13014 ins_encode %{
13015 assert(UseFMA, "Needs FMA instructions support.");
13016 __ fnmaddd(as_FloatRegister($dst$$reg),
13017 as_FloatRegister($src1$$reg),
13018 as_FloatRegister($src2$$reg),
13019 as_FloatRegister($src3$$reg));
13020 %}
13021
13022 ins_pipe(pipe_class_default);
13023 %}
13024
13025 // src1 * src2 - src3
13026 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13027 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13028
13029 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13030
13031 ins_encode %{
13032 assert(UseFMA, "Needs FMA instructions support.");
13033 __ fnmsubs(as_FloatRegister($dst$$reg),
13034 as_FloatRegister($src1$$reg),
13035 as_FloatRegister($src2$$reg),
13036 as_FloatRegister($src3$$reg));
13037 %}
13038
13039 ins_pipe(pipe_class_default);
13040 %}
13041
13042 // src1 * src2 - src3
13043 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13044 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13045
13046 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13047
13048 ins_encode %{
13049 assert(UseFMA, "Needs FMA instructions support.");
13050 // n.b. insn name should be fnmsubd
13051 __ fnmsub(as_FloatRegister($dst$$reg),
13052 as_FloatRegister($src1$$reg),
13053 as_FloatRegister($src2$$reg),
13054 as_FloatRegister($src3$$reg));
13055 %}
13056
13057 ins_pipe(pipe_class_default);
13058 %}
13059
13060 // Math.max(HH)H (half-precision float)
13061 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13062 match(Set dst (MaxHF src1 src2));
13063 format %{ "fmaxh $dst, $src1, $src2" %}
13064 ins_encode %{
13065 __ fmaxh($dst$$FloatRegister,
13066 $src1$$FloatRegister,
13067 $src2$$FloatRegister);
13068 %}
13069 ins_pipe(fp_dop_reg_reg_s);
13070 %}
13071
13072 // Math.min(HH)H (half-precision float)
13073 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13074 match(Set dst (MinHF src1 src2));
13075 format %{ "fminh $dst, $src1, $src2" %}
13076 ins_encode %{
13077 __ fminh($dst$$FloatRegister,
13078 $src1$$FloatRegister,
13079 $src2$$FloatRegister);
13080 %}
13081 ins_pipe(fp_dop_reg_reg_s);
13082 %}
13083
13084 // Math.max(FF)F
13085 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13086 match(Set dst (MaxF src1 src2));
13087
13088 format %{ "fmaxs $dst, $src1, $src2" %}
13089 ins_encode %{
13090 __ fmaxs(as_FloatRegister($dst$$reg),
13091 as_FloatRegister($src1$$reg),
13092 as_FloatRegister($src2$$reg));
13093 %}
13094
13095 ins_pipe(fp_dop_reg_reg_s);
13096 %}
13097
13098 // Math.min(FF)F
13099 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13100 match(Set dst (MinF src1 src2));
13101
13102 format %{ "fmins $dst, $src1, $src2" %}
13103 ins_encode %{
13104 __ fmins(as_FloatRegister($dst$$reg),
13105 as_FloatRegister($src1$$reg),
13106 as_FloatRegister($src2$$reg));
13107 %}
13108
13109 ins_pipe(fp_dop_reg_reg_s);
13110 %}
13111
13112 // Math.max(DD)D
13113 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13114 match(Set dst (MaxD src1 src2));
13115
13116 format %{ "fmaxd $dst, $src1, $src2" %}
13117 ins_encode %{
13118 __ fmaxd(as_FloatRegister($dst$$reg),
13119 as_FloatRegister($src1$$reg),
13120 as_FloatRegister($src2$$reg));
13121 %}
13122
13123 ins_pipe(fp_dop_reg_reg_d);
13124 %}
13125
13126 // Math.min(DD)D
13127 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13128 match(Set dst (MinD src1 src2));
13129
13130 format %{ "fmind $dst, $src1, $src2" %}
13131 ins_encode %{
13132 __ fmind(as_FloatRegister($dst$$reg),
13133 as_FloatRegister($src1$$reg),
13134 as_FloatRegister($src2$$reg));
13135 %}
13136
13137 ins_pipe(fp_dop_reg_reg_d);
13138 %}
13139
13140 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13141 match(Set dst (DivHF src1 src2));
13142 format %{ "fdivh $dst, $src1, $src2" %}
13143 ins_encode %{
13144 __ fdivh($dst$$FloatRegister,
13145 $src1$$FloatRegister,
13146 $src2$$FloatRegister);
13147 %}
13148 ins_pipe(fp_div_s);
13149 %}
13150
13151 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13152 match(Set dst (DivF src1 src2));
13153
13154 ins_cost(INSN_COST * 18);
13155 format %{ "fdivs $dst, $src1, $src2" %}
13156
13157 ins_encode %{
13158 __ fdivs(as_FloatRegister($dst$$reg),
13159 as_FloatRegister($src1$$reg),
13160 as_FloatRegister($src2$$reg));
13161 %}
13162
13163 ins_pipe(fp_div_s);
13164 %}
13165
13166 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13167 match(Set dst (DivD src1 src2));
13168
13169 ins_cost(INSN_COST * 32);
13170 format %{ "fdivd $dst, $src1, $src2" %}
13171
13172 ins_encode %{
13173 __ fdivd(as_FloatRegister($dst$$reg),
13174 as_FloatRegister($src1$$reg),
13175 as_FloatRegister($src2$$reg));
13176 %}
13177
13178 ins_pipe(fp_div_d);
13179 %}
13180
13181 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13182 match(Set dst (NegF src));
13183
13184 ins_cost(INSN_COST * 3);
13185 format %{ "fneg $dst, $src" %}
13186
13187 ins_encode %{
13188 __ fnegs(as_FloatRegister($dst$$reg),
13189 as_FloatRegister($src$$reg));
13190 %}
13191
13192 ins_pipe(fp_uop_s);
13193 %}
13194
13195 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13196 match(Set dst (NegD src));
13197
13198 ins_cost(INSN_COST * 3);
13199 format %{ "fnegd $dst, $src" %}
13200
13201 ins_encode %{
13202 __ fnegd(as_FloatRegister($dst$$reg),
13203 as_FloatRegister($src$$reg));
13204 %}
13205
13206 ins_pipe(fp_uop_d);
13207 %}
13208
13209 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13210 %{
13211 match(Set dst (AbsI src));
13212
13213 effect(KILL cr);
13214 ins_cost(INSN_COST * 2);
13215 format %{ "cmpw $src, zr\n\t"
13216 "cnegw $dst, $src, Assembler::LT\t# int abs"
13217 %}
13218
13219 ins_encode %{
13220 __ cmpw(as_Register($src$$reg), zr);
13221 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13222 %}
13223 ins_pipe(pipe_class_default);
13224 %}
13225
13226 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13227 %{
13228 match(Set dst (AbsL src));
13229
13230 effect(KILL cr);
13231 ins_cost(INSN_COST * 2);
13232 format %{ "cmp $src, zr\n\t"
13233 "cneg $dst, $src, Assembler::LT\t# long abs"
13234 %}
13235
13236 ins_encode %{
13237 __ cmp(as_Register($src$$reg), zr);
13238 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13239 %}
13240 ins_pipe(pipe_class_default);
13241 %}
13242
13243 instruct absF_reg(vRegF dst, vRegF src) %{
13244 match(Set dst (AbsF src));
13245
13246 ins_cost(INSN_COST * 3);
13247 format %{ "fabss $dst, $src" %}
13248 ins_encode %{
13249 __ fabss(as_FloatRegister($dst$$reg),
13250 as_FloatRegister($src$$reg));
13251 %}
13252
13253 ins_pipe(fp_uop_s);
13254 %}
13255
13256 instruct absD_reg(vRegD dst, vRegD src) %{
13257 match(Set dst (AbsD src));
13258
13259 ins_cost(INSN_COST * 3);
13260 format %{ "fabsd $dst, $src" %}
13261 ins_encode %{
13262 __ fabsd(as_FloatRegister($dst$$reg),
13263 as_FloatRegister($src$$reg));
13264 %}
13265
13266 ins_pipe(fp_uop_d);
13267 %}
13268
13269 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13270 match(Set dst (AbsF (SubF src1 src2)));
13271
13272 ins_cost(INSN_COST * 3);
13273 format %{ "fabds $dst, $src1, $src2" %}
13274 ins_encode %{
13275 __ fabds(as_FloatRegister($dst$$reg),
13276 as_FloatRegister($src1$$reg),
13277 as_FloatRegister($src2$$reg));
13278 %}
13279
13280 ins_pipe(fp_uop_s);
13281 %}
13282
13283 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13284 match(Set dst (AbsD (SubD src1 src2)));
13285
13286 ins_cost(INSN_COST * 3);
13287 format %{ "fabdd $dst, $src1, $src2" %}
13288 ins_encode %{
13289 __ fabdd(as_FloatRegister($dst$$reg),
13290 as_FloatRegister($src1$$reg),
13291 as_FloatRegister($src2$$reg));
13292 %}
13293
13294 ins_pipe(fp_uop_d);
13295 %}
13296
13297 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13298 match(Set dst (SqrtD src));
13299
13300 ins_cost(INSN_COST * 50);
13301 format %{ "fsqrtd $dst, $src" %}
13302 ins_encode %{
13303 __ fsqrtd(as_FloatRegister($dst$$reg),
13304 as_FloatRegister($src$$reg));
13305 %}
13306
13307 ins_pipe(fp_div_s);
13308 %}
13309
13310 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13311 match(Set dst (SqrtF src));
13312
13313 ins_cost(INSN_COST * 50);
13314 format %{ "fsqrts $dst, $src" %}
13315 ins_encode %{
13316 __ fsqrts(as_FloatRegister($dst$$reg),
13317 as_FloatRegister($src$$reg));
13318 %}
13319
13320 ins_pipe(fp_div_d);
13321 %}
13322
13323 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13324 match(Set dst (SqrtHF src));
13325 format %{ "fsqrth $dst, $src" %}
13326 ins_encode %{
13327 __ fsqrth($dst$$FloatRegister,
13328 $src$$FloatRegister);
13329 %}
13330 ins_pipe(fp_div_s);
13331 %}
13332
13333 // Math.rint, floor, ceil
13334 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13335 match(Set dst (RoundDoubleMode src rmode));
13336 format %{ "frint $dst, $src, $rmode" %}
13337 ins_encode %{
13338 switch ($rmode$$constant) {
13339 case RoundDoubleModeNode::rmode_rint:
13340 __ frintnd(as_FloatRegister($dst$$reg),
13341 as_FloatRegister($src$$reg));
13342 break;
13343 case RoundDoubleModeNode::rmode_floor:
13344 __ frintmd(as_FloatRegister($dst$$reg),
13345 as_FloatRegister($src$$reg));
13346 break;
13347 case RoundDoubleModeNode::rmode_ceil:
13348 __ frintpd(as_FloatRegister($dst$$reg),
13349 as_FloatRegister($src$$reg));
13350 break;
13351 }
13352 %}
13353 ins_pipe(fp_uop_d);
13354 %}
13355
13356 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13357 match(Set dst (CopySignD src1 (Binary src2 zero)));
13358 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13359 format %{ "CopySignD $dst $src1 $src2" %}
13360 ins_encode %{
13361 FloatRegister dst = as_FloatRegister($dst$$reg),
13362 src1 = as_FloatRegister($src1$$reg),
13363 src2 = as_FloatRegister($src2$$reg),
13364 zero = as_FloatRegister($zero$$reg);
13365 __ fnegd(dst, zero);
13366 __ bsl(dst, __ T8B, src2, src1);
13367 %}
13368 ins_pipe(fp_uop_d);
13369 %}
13370
13371 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13372 match(Set dst (CopySignF src1 src2));
13373 effect(TEMP_DEF dst, USE src1, USE src2);
13374 format %{ "CopySignF $dst $src1 $src2" %}
13375 ins_encode %{
13376 FloatRegister dst = as_FloatRegister($dst$$reg),
13377 src1 = as_FloatRegister($src1$$reg),
13378 src2 = as_FloatRegister($src2$$reg);
13379 __ movi(dst, __ T2S, 0x80, 24);
13380 __ bsl(dst, __ T8B, src2, src1);
13381 %}
13382 ins_pipe(fp_uop_d);
13383 %}
13384
13385 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13386 match(Set dst (SignumD src (Binary zero one)));
13387 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13388 format %{ "signumD $dst, $src" %}
13389 ins_encode %{
13390 FloatRegister src = as_FloatRegister($src$$reg),
13391 dst = as_FloatRegister($dst$$reg),
13392 zero = as_FloatRegister($zero$$reg),
13393 one = as_FloatRegister($one$$reg);
13394 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13395 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13396 // Bit selection instruction gets bit from "one" for each enabled bit in
13397 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13398 // NaN the whole "src" will be copied because "dst" is zero. For all other
13399 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13400 // from "src", and all other bits are copied from 1.0.
13401 __ bsl(dst, __ T8B, one, src);
13402 %}
13403 ins_pipe(fp_uop_d);
13404 %}
13405
13406 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13407 match(Set dst (SignumF src (Binary zero one)));
13408 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13409 format %{ "signumF $dst, $src" %}
13410 ins_encode %{
13411 FloatRegister src = as_FloatRegister($src$$reg),
13412 dst = as_FloatRegister($dst$$reg),
13413 zero = as_FloatRegister($zero$$reg),
13414 one = as_FloatRegister($one$$reg);
13415 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13416 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13417 // Bit selection instruction gets bit from "one" for each enabled bit in
13418 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13419 // NaN the whole "src" will be copied because "dst" is zero. For all other
13420 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13421 // from "src", and all other bits are copied from 1.0.
13422 __ bsl(dst, __ T8B, one, src);
13423 %}
13424 ins_pipe(fp_uop_d);
13425 %}
13426
13427 instruct onspinwait() %{
13428 match(OnSpinWait);
13429 ins_cost(INSN_COST);
13430
13431 format %{ "onspinwait" %}
13432
13433 ins_encode %{
13434 __ spin_wait();
13435 %}
13436 ins_pipe(pipe_class_empty);
13437 %}
13438
13439 // ============================================================================
13440 // Logical Instructions
13441
13442 // Integer Logical Instructions
13443
13444 // And Instructions
13445
13446
13447 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13448 match(Set dst (AndI src1 src2));
13449
13450 format %{ "andw $dst, $src1, $src2\t# int" %}
13451
13452 ins_cost(INSN_COST);
13453 ins_encode %{
13454 __ andw(as_Register($dst$$reg),
13455 as_Register($src1$$reg),
13456 as_Register($src2$$reg));
13457 %}
13458
13459 ins_pipe(ialu_reg_reg);
13460 %}
13461
13462 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13463 match(Set dst (AndI src1 src2));
13464
13465 format %{ "andsw $dst, $src1, $src2\t# int" %}
13466
13467 ins_cost(INSN_COST);
13468 ins_encode %{
13469 __ andw(as_Register($dst$$reg),
13470 as_Register($src1$$reg),
13471 (uint64_t)($src2$$constant));
13472 %}
13473
13474 ins_pipe(ialu_reg_imm);
13475 %}
13476
13477 // Or Instructions
13478
13479 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13480 match(Set dst (OrI src1 src2));
13481
13482 format %{ "orrw $dst, $src1, $src2\t# int" %}
13483
13484 ins_cost(INSN_COST);
13485 ins_encode %{
13486 __ orrw(as_Register($dst$$reg),
13487 as_Register($src1$$reg),
13488 as_Register($src2$$reg));
13489 %}
13490
13491 ins_pipe(ialu_reg_reg);
13492 %}
13493
13494 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13495 match(Set dst (OrI src1 src2));
13496
13497 format %{ "orrw $dst, $src1, $src2\t# int" %}
13498
13499 ins_cost(INSN_COST);
13500 ins_encode %{
13501 __ orrw(as_Register($dst$$reg),
13502 as_Register($src1$$reg),
13503 (uint64_t)($src2$$constant));
13504 %}
13505
13506 ins_pipe(ialu_reg_imm);
13507 %}
13508
13509 // Xor Instructions
13510
13511 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13512 match(Set dst (XorI src1 src2));
13513
13514 format %{ "eorw $dst, $src1, $src2\t# int" %}
13515
13516 ins_cost(INSN_COST);
13517 ins_encode %{
13518 __ eorw(as_Register($dst$$reg),
13519 as_Register($src1$$reg),
13520 as_Register($src2$$reg));
13521 %}
13522
13523 ins_pipe(ialu_reg_reg);
13524 %}
13525
13526 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13527 match(Set dst (XorI src1 src2));
13528
13529 format %{ "eorw $dst, $src1, $src2\t# int" %}
13530
13531 ins_cost(INSN_COST);
13532 ins_encode %{
13533 __ eorw(as_Register($dst$$reg),
13534 as_Register($src1$$reg),
13535 (uint64_t)($src2$$constant));
13536 %}
13537
13538 ins_pipe(ialu_reg_imm);
13539 %}
13540
13541 // Long Logical Instructions
13542 // TODO
13543
13544 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13545 match(Set dst (AndL src1 src2));
13546
13547 format %{ "and $dst, $src1, $src2\t# int" %}
13548
13549 ins_cost(INSN_COST);
13550 ins_encode %{
13551 __ andr(as_Register($dst$$reg),
13552 as_Register($src1$$reg),
13553 as_Register($src2$$reg));
13554 %}
13555
13556 ins_pipe(ialu_reg_reg);
13557 %}
13558
13559 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13560 match(Set dst (AndL src1 src2));
13561
13562 format %{ "and $dst, $src1, $src2\t# int" %}
13563
13564 ins_cost(INSN_COST);
13565 ins_encode %{
13566 __ andr(as_Register($dst$$reg),
13567 as_Register($src1$$reg),
13568 (uint64_t)($src2$$constant));
13569 %}
13570
13571 ins_pipe(ialu_reg_imm);
13572 %}
13573
13574 // Or Instructions
13575
13576 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13577 match(Set dst (OrL src1 src2));
13578
13579 format %{ "orr $dst, $src1, $src2\t# int" %}
13580
13581 ins_cost(INSN_COST);
13582 ins_encode %{
13583 __ orr(as_Register($dst$$reg),
13584 as_Register($src1$$reg),
13585 as_Register($src2$$reg));
13586 %}
13587
13588 ins_pipe(ialu_reg_reg);
13589 %}
13590
13591 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13592 match(Set dst (OrL src1 src2));
13593
13594 format %{ "orr $dst, $src1, $src2\t# int" %}
13595
13596 ins_cost(INSN_COST);
13597 ins_encode %{
13598 __ orr(as_Register($dst$$reg),
13599 as_Register($src1$$reg),
13600 (uint64_t)($src2$$constant));
13601 %}
13602
13603 ins_pipe(ialu_reg_imm);
13604 %}
13605
13606 // Xor Instructions
13607
13608 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13609 match(Set dst (XorL src1 src2));
13610
13611 format %{ "eor $dst, $src1, $src2\t# int" %}
13612
13613 ins_cost(INSN_COST);
13614 ins_encode %{
13615 __ eor(as_Register($dst$$reg),
13616 as_Register($src1$$reg),
13617 as_Register($src2$$reg));
13618 %}
13619
13620 ins_pipe(ialu_reg_reg);
13621 %}
13622
13623 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13624 match(Set dst (XorL src1 src2));
13625
13626 ins_cost(INSN_COST);
13627 format %{ "eor $dst, $src1, $src2\t# int" %}
13628
13629 ins_encode %{
13630 __ eor(as_Register($dst$$reg),
13631 as_Register($src1$$reg),
13632 (uint64_t)($src2$$constant));
13633 %}
13634
13635 ins_pipe(ialu_reg_imm);
13636 %}
13637
13638 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13639 %{
13640 match(Set dst (ConvI2L src));
13641
13642 ins_cost(INSN_COST);
13643 format %{ "sxtw $dst, $src\t# i2l" %}
13644 ins_encode %{
13645 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13646 %}
13647 ins_pipe(ialu_reg_shift);
13648 %}
13649
13650 // this pattern occurs in bigmath arithmetic
13651 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13652 %{
13653 match(Set dst (AndL (ConvI2L src) mask));
13654
13655 ins_cost(INSN_COST);
13656 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13657 ins_encode %{
13658 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13659 %}
13660
13661 ins_pipe(ialu_reg_shift);
13662 %}
13663
13664 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13665 match(Set dst (ConvL2I src));
13666
13667 ins_cost(INSN_COST);
13668 format %{ "movw $dst, $src \t// l2i" %}
13669
13670 ins_encode %{
13671 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13672 %}
13673
13674 ins_pipe(ialu_reg);
13675 %}
13676
13677 instruct convD2F_reg(vRegF dst, vRegD src) %{
13678 match(Set dst (ConvD2F src));
13679
13680 ins_cost(INSN_COST * 5);
13681 format %{ "fcvtd $dst, $src \t// d2f" %}
13682
13683 ins_encode %{
13684 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13685 %}
13686
13687 ins_pipe(fp_d2f);
13688 %}
13689
13690 instruct convF2D_reg(vRegD dst, vRegF src) %{
13691 match(Set dst (ConvF2D src));
13692
13693 ins_cost(INSN_COST * 5);
13694 format %{ "fcvts $dst, $src \t// f2d" %}
13695
13696 ins_encode %{
13697 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13698 %}
13699
13700 ins_pipe(fp_f2d);
13701 %}
13702
13703 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13704 match(Set dst (ConvF2I src));
13705
13706 ins_cost(INSN_COST * 5);
13707 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13708
13709 ins_encode %{
13710 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13711 %}
13712
13713 ins_pipe(fp_f2i);
13714 %}
13715
13716 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13717 match(Set dst (ConvF2L src));
13718
13719 ins_cost(INSN_COST * 5);
13720 format %{ "fcvtzs $dst, $src \t// f2l" %}
13721
13722 ins_encode %{
13723 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13724 %}
13725
13726 ins_pipe(fp_f2l);
13727 %}
13728
13729 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13730 match(Set dst (ConvF2HF src));
13731 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13732 "smov $dst, $tmp\t# move result from $tmp to $dst"
13733 %}
13734 effect(TEMP tmp);
13735 ins_encode %{
13736 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13737 %}
13738 ins_pipe(pipe_slow);
13739 %}
13740
13741 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13742 match(Set dst (ConvHF2F src));
13743 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13744 "fcvt $dst, $tmp\t# convert half to single precision"
13745 %}
13746 effect(TEMP tmp);
13747 ins_encode %{
13748 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13749 %}
13750 ins_pipe(pipe_slow);
13751 %}
13752
13753 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13754 match(Set dst (ConvI2F src));
13755
13756 ins_cost(INSN_COST * 5);
13757 format %{ "scvtfws $dst, $src \t// i2f" %}
13758
13759 ins_encode %{
13760 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13761 %}
13762
13763 ins_pipe(fp_i2f);
13764 %}
13765
13766 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13767 match(Set dst (ConvL2F src));
13768
13769 ins_cost(INSN_COST * 5);
13770 format %{ "scvtfs $dst, $src \t// l2f" %}
13771
13772 ins_encode %{
13773 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13774 %}
13775
13776 ins_pipe(fp_l2f);
13777 %}
13778
13779 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13780 match(Set dst (ConvD2I src));
13781
13782 ins_cost(INSN_COST * 5);
13783 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13784
13785 ins_encode %{
13786 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13787 %}
13788
13789 ins_pipe(fp_d2i);
13790 %}
13791
13792 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13793 match(Set dst (ConvD2L src));
13794
13795 ins_cost(INSN_COST * 5);
13796 format %{ "fcvtzd $dst, $src \t// d2l" %}
13797
13798 ins_encode %{
13799 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13800 %}
13801
13802 ins_pipe(fp_d2l);
13803 %}
13804
13805 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13806 match(Set dst (ConvI2D src));
13807
13808 ins_cost(INSN_COST * 5);
13809 format %{ "scvtfwd $dst, $src \t// i2d" %}
13810
13811 ins_encode %{
13812 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13813 %}
13814
13815 ins_pipe(fp_i2d);
13816 %}
13817
13818 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13819 match(Set dst (ConvL2D src));
13820
13821 ins_cost(INSN_COST * 5);
13822 format %{ "scvtfd $dst, $src \t// l2d" %}
13823
13824 ins_encode %{
13825 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13826 %}
13827
13828 ins_pipe(fp_l2d);
13829 %}
13830
13831 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13832 %{
13833 match(Set dst (RoundD src));
13834 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13835 format %{ "java_round_double $dst,$src"%}
13836 ins_encode %{
13837 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13838 as_FloatRegister($ftmp$$reg));
13839 %}
13840 ins_pipe(pipe_slow);
13841 %}
13842
13843 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13844 %{
13845 match(Set dst (RoundF src));
13846 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13847 format %{ "java_round_float $dst,$src"%}
13848 ins_encode %{
13849 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13850 as_FloatRegister($ftmp$$reg));
13851 %}
13852 ins_pipe(pipe_slow);
13853 %}
13854
13855 // stack <-> reg and reg <-> reg shuffles with no conversion
13856
13857 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13858
13859 match(Set dst (MoveF2I src));
13860
13861 effect(DEF dst, USE src);
13862
13863 ins_cost(4 * INSN_COST);
13864
13865 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13866
13867 ins_encode %{
13868 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13869 %}
13870
13871 ins_pipe(iload_reg_reg);
13872
13873 %}
13874
13875 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13876
13877 match(Set dst (MoveI2F src));
13878
13879 effect(DEF dst, USE src);
13880
13881 ins_cost(4 * INSN_COST);
13882
13883 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13884
13885 ins_encode %{
13886 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13887 %}
13888
13889 ins_pipe(pipe_class_memory);
13890
13891 %}
13892
13893 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13894
13895 match(Set dst (MoveD2L src));
13896
13897 effect(DEF dst, USE src);
13898
13899 ins_cost(4 * INSN_COST);
13900
13901 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13902
13903 ins_encode %{
13904 __ ldr($dst$$Register, Address(sp, $src$$disp));
13905 %}
13906
13907 ins_pipe(iload_reg_reg);
13908
13909 %}
13910
13911 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13912
13913 match(Set dst (MoveL2D src));
13914
13915 effect(DEF dst, USE src);
13916
13917 ins_cost(4 * INSN_COST);
13918
13919 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13920
13921 ins_encode %{
13922 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13923 %}
13924
13925 ins_pipe(pipe_class_memory);
13926
13927 %}
13928
13929 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13930
13931 match(Set dst (MoveF2I src));
13932
13933 effect(DEF dst, USE src);
13934
13935 ins_cost(INSN_COST);
13936
13937 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13938
13939 ins_encode %{
13940 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13941 %}
13942
13943 ins_pipe(pipe_class_memory);
13944
13945 %}
13946
13947 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13948
13949 match(Set dst (MoveI2F src));
13950
13951 effect(DEF dst, USE src);
13952
13953 ins_cost(INSN_COST);
13954
13955 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13956
13957 ins_encode %{
13958 __ strw($src$$Register, Address(sp, $dst$$disp));
13959 %}
13960
13961 ins_pipe(istore_reg_reg);
13962
13963 %}
13964
13965 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13966
13967 match(Set dst (MoveD2L src));
13968
13969 effect(DEF dst, USE src);
13970
13971 ins_cost(INSN_COST);
13972
13973 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13974
13975 ins_encode %{
13976 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13977 %}
13978
13979 ins_pipe(pipe_class_memory);
13980
13981 %}
13982
13983 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13984
13985 match(Set dst (MoveL2D src));
13986
13987 effect(DEF dst, USE src);
13988
13989 ins_cost(INSN_COST);
13990
13991 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13992
13993 ins_encode %{
13994 __ str($src$$Register, Address(sp, $dst$$disp));
13995 %}
13996
13997 ins_pipe(istore_reg_reg);
13998
13999 %}
14000
14001 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14002
14003 match(Set dst (MoveF2I src));
14004
14005 effect(DEF dst, USE src);
14006
14007 ins_cost(INSN_COST);
14008
14009 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14010
14011 ins_encode %{
14012 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14013 %}
14014
14015 ins_pipe(fp_f2i);
14016
14017 %}
14018
14019 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14020
14021 match(Set dst (MoveI2F src));
14022
14023 effect(DEF dst, USE src);
14024
14025 ins_cost(INSN_COST);
14026
14027 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14028
14029 ins_encode %{
14030 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14031 %}
14032
14033 ins_pipe(fp_i2f);
14034
14035 %}
14036
14037 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14038
14039 match(Set dst (MoveD2L src));
14040
14041 effect(DEF dst, USE src);
14042
14043 ins_cost(INSN_COST);
14044
14045 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14046
14047 ins_encode %{
14048 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14049 %}
14050
14051 ins_pipe(fp_d2l);
14052
14053 %}
14054
14055 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14056
14057 match(Set dst (MoveL2D src));
14058
14059 effect(DEF dst, USE src);
14060
14061 ins_cost(INSN_COST);
14062
14063 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14064
14065 ins_encode %{
14066 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14067 %}
14068
14069 ins_pipe(fp_l2d);
14070
14071 %}
14072
14073 // ============================================================================
14074 // clearing of an array
14075
14076 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14077 %{
14078 match(Set dummy (ClearArray cnt base));
14079 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14080
14081 ins_cost(4 * INSN_COST);
14082 format %{ "ClearArray $cnt, $base" %}
14083
14084 ins_encode %{
14085 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14086 if (tpc == nullptr) {
14087 ciEnv::current()->record_failure("CodeCache is full");
14088 return;
14089 }
14090 %}
14091
14092 ins_pipe(pipe_class_memory);
14093 %}
14094
14095 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14096 %{
14097 predicate((uint64_t)n->in(2)->get_long()
14098 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14099 match(Set dummy (ClearArray cnt base));
14100 effect(TEMP temp, USE_KILL base, KILL cr);
14101
14102 ins_cost(4 * INSN_COST);
14103 format %{ "ClearArray $cnt, $base" %}
14104
14105 ins_encode %{
14106 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14107 if (tpc == nullptr) {
14108 ciEnv::current()->record_failure("CodeCache is full");
14109 return;
14110 }
14111 %}
14112
14113 ins_pipe(pipe_class_memory);
14114 %}
14115
14116 // ============================================================================
14117 // Overflow Math Instructions
14118
14119 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14120 %{
14121 match(Set cr (OverflowAddI op1 op2));
14122
14123 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14124 ins_cost(INSN_COST);
14125 ins_encode %{
14126 __ cmnw($op1$$Register, $op2$$Register);
14127 %}
14128
14129 ins_pipe(icmp_reg_reg);
14130 %}
14131
14132 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14133 %{
14134 match(Set cr (OverflowAddI op1 op2));
14135
14136 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14137 ins_cost(INSN_COST);
14138 ins_encode %{
14139 __ cmnw($op1$$Register, $op2$$constant);
14140 %}
14141
14142 ins_pipe(icmp_reg_imm);
14143 %}
14144
14145 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14146 %{
14147 match(Set cr (OverflowAddL op1 op2));
14148
14149 format %{ "cmn $op1, $op2\t# overflow check long" %}
14150 ins_cost(INSN_COST);
14151 ins_encode %{
14152 __ cmn($op1$$Register, $op2$$Register);
14153 %}
14154
14155 ins_pipe(icmp_reg_reg);
14156 %}
14157
14158 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14159 %{
14160 match(Set cr (OverflowAddL op1 op2));
14161
14162 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14163 ins_cost(INSN_COST);
14164 ins_encode %{
14165 __ adds(zr, $op1$$Register, $op2$$constant);
14166 %}
14167
14168 ins_pipe(icmp_reg_imm);
14169 %}
14170
14171 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14172 %{
14173 match(Set cr (OverflowSubI op1 op2));
14174
14175 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14176 ins_cost(INSN_COST);
14177 ins_encode %{
14178 __ cmpw($op1$$Register, $op2$$Register);
14179 %}
14180
14181 ins_pipe(icmp_reg_reg);
14182 %}
14183
14184 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14185 %{
14186 match(Set cr (OverflowSubI op1 op2));
14187
14188 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14189 ins_cost(INSN_COST);
14190 ins_encode %{
14191 __ cmpw($op1$$Register, $op2$$constant);
14192 %}
14193
14194 ins_pipe(icmp_reg_imm);
14195 %}
14196
14197 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14198 %{
14199 match(Set cr (OverflowSubL op1 op2));
14200
14201 format %{ "cmp $op1, $op2\t# overflow check long" %}
14202 ins_cost(INSN_COST);
14203 ins_encode %{
14204 __ cmp($op1$$Register, $op2$$Register);
14205 %}
14206
14207 ins_pipe(icmp_reg_reg);
14208 %}
14209
14210 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14211 %{
14212 match(Set cr (OverflowSubL op1 op2));
14213
14214 format %{ "cmp $op1, $op2\t# overflow check long" %}
14215 ins_cost(INSN_COST);
14216 ins_encode %{
14217 __ subs(zr, $op1$$Register, $op2$$constant);
14218 %}
14219
14220 ins_pipe(icmp_reg_imm);
14221 %}
14222
14223 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14224 %{
14225 match(Set cr (OverflowSubI zero op1));
14226
14227 format %{ "cmpw zr, $op1\t# overflow check int" %}
14228 ins_cost(INSN_COST);
14229 ins_encode %{
14230 __ cmpw(zr, $op1$$Register);
14231 %}
14232
14233 ins_pipe(icmp_reg_imm);
14234 %}
14235
14236 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14237 %{
14238 match(Set cr (OverflowSubL zero op1));
14239
14240 format %{ "cmp zr, $op1\t# overflow check long" %}
14241 ins_cost(INSN_COST);
14242 ins_encode %{
14243 __ cmp(zr, $op1$$Register);
14244 %}
14245
14246 ins_pipe(icmp_reg_imm);
14247 %}
14248
14249 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14250 %{
14251 match(Set cr (OverflowMulI op1 op2));
14252
14253 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14254 "cmp rscratch1, rscratch1, sxtw\n\t"
14255 "movw rscratch1, #0x80000000\n\t"
14256 "cselw rscratch1, rscratch1, zr, NE\n\t"
14257 "cmpw rscratch1, #1" %}
14258 ins_cost(5 * INSN_COST);
14259 ins_encode %{
14260 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14261 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14262 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14263 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14264 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14265 %}
14266
14267 ins_pipe(pipe_slow);
14268 %}
14269
14270 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14271 %{
14272 match(If cmp (OverflowMulI op1 op2));
14273 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14274 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14275 effect(USE labl, KILL cr);
14276
14277 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14278 "cmp rscratch1, rscratch1, sxtw\n\t"
14279 "b$cmp $labl" %}
14280 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14281 ins_encode %{
14282 Label* L = $labl$$label;
14283 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14284 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14285 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14286 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14287 %}
14288
14289 ins_pipe(pipe_serial);
14290 %}
14291
14292 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14293 %{
14294 match(Set cr (OverflowMulL op1 op2));
14295
14296 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14297 "smulh rscratch2, $op1, $op2\n\t"
14298 "cmp rscratch2, rscratch1, ASR #63\n\t"
14299 "movw rscratch1, #0x80000000\n\t"
14300 "cselw rscratch1, rscratch1, zr, NE\n\t"
14301 "cmpw rscratch1, #1" %}
14302 ins_cost(6 * INSN_COST);
14303 ins_encode %{
14304 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14305 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14306 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14307 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14308 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14309 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14310 %}
14311
14312 ins_pipe(pipe_slow);
14313 %}
14314
14315 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14316 %{
14317 match(If cmp (OverflowMulL op1 op2));
14318 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14319 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14320 effect(USE labl, KILL cr);
14321
14322 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14323 "smulh rscratch2, $op1, $op2\n\t"
14324 "cmp rscratch2, rscratch1, ASR #63\n\t"
14325 "b$cmp $labl" %}
14326 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14327 ins_encode %{
14328 Label* L = $labl$$label;
14329 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14330 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14331 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14332 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14333 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14334 %}
14335
14336 ins_pipe(pipe_serial);
14337 %}
14338
14339 // ============================================================================
14340 // Compare Instructions
14341
14342 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14343 %{
14344 match(Set cr (CmpI op1 op2));
14345
14346 effect(DEF cr, USE op1, USE op2);
14347
14348 ins_cost(INSN_COST);
14349 format %{ "cmpw $op1, $op2" %}
14350
14351 ins_encode(aarch64_enc_cmpw(op1, op2));
14352
14353 ins_pipe(icmp_reg_reg);
14354 %}
14355
14356 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14357 %{
14358 match(Set cr (CmpI op1 zero));
14359
14360 effect(DEF cr, USE op1);
14361
14362 ins_cost(INSN_COST);
14363 format %{ "cmpw $op1, 0" %}
14364
14365 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14366
14367 ins_pipe(icmp_reg_imm);
14368 %}
14369
14370 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14371 %{
14372 match(Set cr (CmpI op1 op2));
14373
14374 effect(DEF cr, USE op1);
14375
14376 ins_cost(INSN_COST);
14377 format %{ "cmpw $op1, $op2" %}
14378
14379 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14380
14381 ins_pipe(icmp_reg_imm);
14382 %}
14383
14384 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14385 %{
14386 match(Set cr (CmpI op1 op2));
14387
14388 effect(DEF cr, USE op1);
14389
14390 ins_cost(INSN_COST * 2);
14391 format %{ "cmpw $op1, $op2" %}
14392
14393 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14394
14395 ins_pipe(icmp_reg_imm);
14396 %}
14397
14398 // Unsigned compare Instructions; really, same as signed compare
14399 // except it should only be used to feed an If or a CMovI which takes a
14400 // cmpOpU.
14401
14402 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14403 %{
14404 match(Set cr (CmpU op1 op2));
14405
14406 effect(DEF cr, USE op1, USE op2);
14407
14408 ins_cost(INSN_COST);
14409 format %{ "cmpw $op1, $op2\t# unsigned" %}
14410
14411 ins_encode(aarch64_enc_cmpw(op1, op2));
14412
14413 ins_pipe(icmp_reg_reg);
14414 %}
14415
14416 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14417 %{
14418 match(Set cr (CmpU op1 zero));
14419
14420 effect(DEF cr, USE op1);
14421
14422 ins_cost(INSN_COST);
14423 format %{ "cmpw $op1, #0\t# unsigned" %}
14424
14425 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14426
14427 ins_pipe(icmp_reg_imm);
14428 %}
14429
14430 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14431 %{
14432 match(Set cr (CmpU op1 op2));
14433
14434 effect(DEF cr, USE op1);
14435
14436 ins_cost(INSN_COST);
14437 format %{ "cmpw $op1, $op2\t# unsigned" %}
14438
14439 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14440
14441 ins_pipe(icmp_reg_imm);
14442 %}
14443
14444 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14445 %{
14446 match(Set cr (CmpU op1 op2));
14447
14448 effect(DEF cr, USE op1);
14449
14450 ins_cost(INSN_COST * 2);
14451 format %{ "cmpw $op1, $op2\t# unsigned" %}
14452
14453 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14454
14455 ins_pipe(icmp_reg_imm);
14456 %}
14457
14458 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14459 %{
14460 match(Set cr (CmpL op1 op2));
14461
14462 effect(DEF cr, USE op1, USE op2);
14463
14464 ins_cost(INSN_COST);
14465 format %{ "cmp $op1, $op2" %}
14466
14467 ins_encode(aarch64_enc_cmp(op1, op2));
14468
14469 ins_pipe(icmp_reg_reg);
14470 %}
14471
14472 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14473 %{
14474 match(Set cr (CmpL op1 zero));
14475
14476 effect(DEF cr, USE op1);
14477
14478 ins_cost(INSN_COST);
14479 format %{ "tst $op1" %}
14480
14481 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14482
14483 ins_pipe(icmp_reg_imm);
14484 %}
14485
14486 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14487 %{
14488 match(Set cr (CmpL op1 op2));
14489
14490 effect(DEF cr, USE op1);
14491
14492 ins_cost(INSN_COST);
14493 format %{ "cmp $op1, $op2" %}
14494
14495 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14496
14497 ins_pipe(icmp_reg_imm);
14498 %}
14499
14500 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14501 %{
14502 match(Set cr (CmpL op1 op2));
14503
14504 effect(DEF cr, USE op1);
14505
14506 ins_cost(INSN_COST * 2);
14507 format %{ "cmp $op1, $op2" %}
14508
14509 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14510
14511 ins_pipe(icmp_reg_imm);
14512 %}
14513
14514 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14515 %{
14516 match(Set cr (CmpUL op1 op2));
14517
14518 effect(DEF cr, USE op1, USE op2);
14519
14520 ins_cost(INSN_COST);
14521 format %{ "cmp $op1, $op2" %}
14522
14523 ins_encode(aarch64_enc_cmp(op1, op2));
14524
14525 ins_pipe(icmp_reg_reg);
14526 %}
14527
14528 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14529 %{
14530 match(Set cr (CmpUL op1 zero));
14531
14532 effect(DEF cr, USE op1);
14533
14534 ins_cost(INSN_COST);
14535 format %{ "tst $op1" %}
14536
14537 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14538
14539 ins_pipe(icmp_reg_imm);
14540 %}
14541
14542 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14543 %{
14544 match(Set cr (CmpUL op1 op2));
14545
14546 effect(DEF cr, USE op1);
14547
14548 ins_cost(INSN_COST);
14549 format %{ "cmp $op1, $op2" %}
14550
14551 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14552
14553 ins_pipe(icmp_reg_imm);
14554 %}
14555
14556 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14557 %{
14558 match(Set cr (CmpUL op1 op2));
14559
14560 effect(DEF cr, USE op1);
14561
14562 ins_cost(INSN_COST * 2);
14563 format %{ "cmp $op1, $op2" %}
14564
14565 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14566
14567 ins_pipe(icmp_reg_imm);
14568 %}
14569
14570 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14571 %{
14572 match(Set cr (CmpP op1 op2));
14573
14574 effect(DEF cr, USE op1, USE op2);
14575
14576 ins_cost(INSN_COST);
14577 format %{ "cmp $op1, $op2\t // ptr" %}
14578
14579 ins_encode(aarch64_enc_cmpp(op1, op2));
14580
14581 ins_pipe(icmp_reg_reg);
14582 %}
14583
14584 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14585 %{
14586 match(Set cr (CmpN op1 op2));
14587
14588 effect(DEF cr, USE op1, USE op2);
14589
14590 ins_cost(INSN_COST);
14591 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14592
14593 ins_encode(aarch64_enc_cmpn(op1, op2));
14594
14595 ins_pipe(icmp_reg_reg);
14596 %}
14597
14598 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14599 %{
14600 match(Set cr (CmpP op1 zero));
14601
14602 effect(DEF cr, USE op1, USE zero);
14603
14604 ins_cost(INSN_COST);
14605 format %{ "cmp $op1, 0\t // ptr" %}
14606
14607 ins_encode(aarch64_enc_testp(op1));
14608
14609 ins_pipe(icmp_reg_imm);
14610 %}
14611
14612 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14613 %{
14614 match(Set cr (CmpN op1 zero));
14615
14616 effect(DEF cr, USE op1, USE zero);
14617
14618 ins_cost(INSN_COST);
14619 format %{ "cmp $op1, 0\t // compressed ptr" %}
14620
14621 ins_encode(aarch64_enc_testn(op1));
14622
14623 ins_pipe(icmp_reg_imm);
14624 %}
14625
14626 // FP comparisons
14627 //
14628 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14629 // using normal cmpOp. See declaration of rFlagsReg for details.
14630
14631 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14632 %{
14633 match(Set cr (CmpF src1 src2));
14634
14635 ins_cost(3 * INSN_COST);
14636 format %{ "fcmps $src1, $src2" %}
14637
14638 ins_encode %{
14639 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14640 %}
14641
14642 ins_pipe(pipe_class_compare);
14643 %}
14644
14645 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14646 %{
14647 match(Set cr (CmpF src1 src2));
14648
14649 ins_cost(3 * INSN_COST);
14650 format %{ "fcmps $src1, 0.0" %}
14651
14652 ins_encode %{
14653 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14654 %}
14655
14656 ins_pipe(pipe_class_compare);
14657 %}
14658 // FROM HERE
14659
14660 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14661 %{
14662 match(Set cr (CmpD src1 src2));
14663
14664 ins_cost(3 * INSN_COST);
14665 format %{ "fcmpd $src1, $src2" %}
14666
14667 ins_encode %{
14668 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14669 %}
14670
14671 ins_pipe(pipe_class_compare);
14672 %}
14673
14674 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14675 %{
14676 match(Set cr (CmpD src1 src2));
14677
14678 ins_cost(3 * INSN_COST);
14679 format %{ "fcmpd $src1, 0.0" %}
14680
14681 ins_encode %{
14682 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14683 %}
14684
14685 ins_pipe(pipe_class_compare);
14686 %}
14687
14688 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14689 %{
14690 match(Set dst (CmpF3 src1 src2));
14691 effect(KILL cr);
14692
14693 ins_cost(5 * INSN_COST);
14694 format %{ "fcmps $src1, $src2\n\t"
14695 "csinvw($dst, zr, zr, eq\n\t"
14696 "csnegw($dst, $dst, $dst, lt)"
14697 %}
14698
14699 ins_encode %{
14700 Label done;
14701 FloatRegister s1 = as_FloatRegister($src1$$reg);
14702 FloatRegister s2 = as_FloatRegister($src2$$reg);
14703 Register d = as_Register($dst$$reg);
14704 __ fcmps(s1, s2);
14705 // installs 0 if EQ else -1
14706 __ csinvw(d, zr, zr, Assembler::EQ);
14707 // keeps -1 if less or unordered else installs 1
14708 __ csnegw(d, d, d, Assembler::LT);
14709 __ bind(done);
14710 %}
14711
14712 ins_pipe(pipe_class_default);
14713
14714 %}
14715
14716 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14717 %{
14718 match(Set dst (CmpD3 src1 src2));
14719 effect(KILL cr);
14720
14721 ins_cost(5 * INSN_COST);
14722 format %{ "fcmpd $src1, $src2\n\t"
14723 "csinvw($dst, zr, zr, eq\n\t"
14724 "csnegw($dst, $dst, $dst, lt)"
14725 %}
14726
14727 ins_encode %{
14728 Label done;
14729 FloatRegister s1 = as_FloatRegister($src1$$reg);
14730 FloatRegister s2 = as_FloatRegister($src2$$reg);
14731 Register d = as_Register($dst$$reg);
14732 __ fcmpd(s1, s2);
14733 // installs 0 if EQ else -1
14734 __ csinvw(d, zr, zr, Assembler::EQ);
14735 // keeps -1 if less or unordered else installs 1
14736 __ csnegw(d, d, d, Assembler::LT);
14737 __ bind(done);
14738 %}
14739 ins_pipe(pipe_class_default);
14740
14741 %}
14742
14743 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14744 %{
14745 match(Set dst (CmpF3 src1 zero));
14746 effect(KILL cr);
14747
14748 ins_cost(5 * INSN_COST);
14749 format %{ "fcmps $src1, 0.0\n\t"
14750 "csinvw($dst, zr, zr, eq\n\t"
14751 "csnegw($dst, $dst, $dst, lt)"
14752 %}
14753
14754 ins_encode %{
14755 Label done;
14756 FloatRegister s1 = as_FloatRegister($src1$$reg);
14757 Register d = as_Register($dst$$reg);
14758 __ fcmps(s1, 0.0);
14759 // installs 0 if EQ else -1
14760 __ csinvw(d, zr, zr, Assembler::EQ);
14761 // keeps -1 if less or unordered else installs 1
14762 __ csnegw(d, d, d, Assembler::LT);
14763 __ bind(done);
14764 %}
14765
14766 ins_pipe(pipe_class_default);
14767
14768 %}
14769
14770 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14771 %{
14772 match(Set dst (CmpD3 src1 zero));
14773 effect(KILL cr);
14774
14775 ins_cost(5 * INSN_COST);
14776 format %{ "fcmpd $src1, 0.0\n\t"
14777 "csinvw($dst, zr, zr, eq\n\t"
14778 "csnegw($dst, $dst, $dst, lt)"
14779 %}
14780
14781 ins_encode %{
14782 Label done;
14783 FloatRegister s1 = as_FloatRegister($src1$$reg);
14784 Register d = as_Register($dst$$reg);
14785 __ fcmpd(s1, 0.0);
14786 // installs 0 if EQ else -1
14787 __ csinvw(d, zr, zr, Assembler::EQ);
14788 // keeps -1 if less or unordered else installs 1
14789 __ csnegw(d, d, d, Assembler::LT);
14790 __ bind(done);
14791 %}
14792 ins_pipe(pipe_class_default);
14793
14794 %}
14795
14796 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14797 %{
14798 match(Set dst (CmpLTMask p q));
14799 effect(KILL cr);
14800
14801 ins_cost(3 * INSN_COST);
14802
14803 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14804 "csetw $dst, lt\n\t"
14805 "subw $dst, zr, $dst"
14806 %}
14807
14808 ins_encode %{
14809 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14810 __ csetw(as_Register($dst$$reg), Assembler::LT);
14811 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14812 %}
14813
14814 ins_pipe(ialu_reg_reg);
14815 %}
14816
14817 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14818 %{
14819 match(Set dst (CmpLTMask src zero));
14820 effect(KILL cr);
14821
14822 ins_cost(INSN_COST);
14823
14824 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14825
14826 ins_encode %{
14827 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14828 %}
14829
14830 ins_pipe(ialu_reg_shift);
14831 %}
14832
14833 // ============================================================================
14834 // Max and Min
14835
14836 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14837
14838 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14839 %{
14840 effect(DEF cr, USE src);
14841 ins_cost(INSN_COST);
14842 format %{ "cmpw $src, 0" %}
14843
14844 ins_encode %{
14845 __ cmpw($src$$Register, 0);
14846 %}
14847 ins_pipe(icmp_reg_imm);
14848 %}
14849
14850 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14851 %{
14852 match(Set dst (MinI src1 src2));
14853 ins_cost(INSN_COST * 3);
14854
14855 expand %{
14856 rFlagsReg cr;
14857 compI_reg_reg(cr, src1, src2);
14858 cmovI_reg_reg_lt(dst, src1, src2, cr);
14859 %}
14860 %}
14861
14862 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14863 %{
14864 match(Set dst (MaxI src1 src2));
14865 ins_cost(INSN_COST * 3);
14866
14867 expand %{
14868 rFlagsReg cr;
14869 compI_reg_reg(cr, src1, src2);
14870 cmovI_reg_reg_gt(dst, src1, src2, cr);
14871 %}
14872 %}
14873
14874
14875 // ============================================================================
14876 // Branch Instructions
14877
14878 // Direct Branch.
14879 instruct branch(label lbl)
14880 %{
14881 match(Goto);
14882
14883 effect(USE lbl);
14884
14885 ins_cost(BRANCH_COST);
14886 format %{ "b $lbl" %}
14887
14888 ins_encode(aarch64_enc_b(lbl));
14889
14890 ins_pipe(pipe_branch);
14891 %}
14892
14893 // Conditional Near Branch
14894 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14895 %{
14896 // Same match rule as `branchConFar'.
14897 match(If cmp cr);
14898
14899 effect(USE lbl);
14900
14901 ins_cost(BRANCH_COST);
14902 // If set to 1 this indicates that the current instruction is a
14903 // short variant of a long branch. This avoids using this
14904 // instruction in first-pass matching. It will then only be used in
14905 // the `Shorten_branches' pass.
14906 // ins_short_branch(1);
14907 format %{ "b$cmp $lbl" %}
14908
14909 ins_encode(aarch64_enc_br_con(cmp, lbl));
14910
14911 ins_pipe(pipe_branch_cond);
14912 %}
14913
14914 // Conditional Near Branch Unsigned
14915 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14916 %{
14917 // Same match rule as `branchConFar'.
14918 match(If cmp cr);
14919
14920 effect(USE lbl);
14921
14922 ins_cost(BRANCH_COST);
14923 // If set to 1 this indicates that the current instruction is a
14924 // short variant of a long branch. This avoids using this
14925 // instruction in first-pass matching. It will then only be used in
14926 // the `Shorten_branches' pass.
14927 // ins_short_branch(1);
14928 format %{ "b$cmp $lbl\t# unsigned" %}
14929
14930 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14931
14932 ins_pipe(pipe_branch_cond);
14933 %}
14934
14935 // Make use of CBZ and CBNZ. These instructions, as well as being
14936 // shorter than (cmp; branch), have the additional benefit of not
14937 // killing the flags.
14938
14939 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14940 match(If cmp (CmpI op1 op2));
14941 effect(USE labl);
14942
14943 ins_cost(BRANCH_COST);
14944 format %{ "cbw$cmp $op1, $labl" %}
14945 ins_encode %{
14946 Label* L = $labl$$label;
14947 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14948 if (cond == Assembler::EQ)
14949 __ cbzw($op1$$Register, *L);
14950 else
14951 __ cbnzw($op1$$Register, *L);
14952 %}
14953 ins_pipe(pipe_cmp_branch);
14954 %}
14955
14956 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14957 match(If cmp (CmpL op1 op2));
14958 effect(USE labl);
14959
14960 ins_cost(BRANCH_COST);
14961 format %{ "cb$cmp $op1, $labl" %}
14962 ins_encode %{
14963 Label* L = $labl$$label;
14964 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14965 if (cond == Assembler::EQ)
14966 __ cbz($op1$$Register, *L);
14967 else
14968 __ cbnz($op1$$Register, *L);
14969 %}
14970 ins_pipe(pipe_cmp_branch);
14971 %}
14972
14973 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14974 match(If cmp (CmpP op1 op2));
14975 effect(USE labl);
14976
14977 ins_cost(BRANCH_COST);
14978 format %{ "cb$cmp $op1, $labl" %}
14979 ins_encode %{
14980 Label* L = $labl$$label;
14981 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14982 if (cond == Assembler::EQ)
14983 __ cbz($op1$$Register, *L);
14984 else
14985 __ cbnz($op1$$Register, *L);
14986 %}
14987 ins_pipe(pipe_cmp_branch);
14988 %}
14989
14990 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14991 match(If cmp (CmpN op1 op2));
14992 effect(USE labl);
14993
14994 ins_cost(BRANCH_COST);
14995 format %{ "cbw$cmp $op1, $labl" %}
14996 ins_encode %{
14997 Label* L = $labl$$label;
14998 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14999 if (cond == Assembler::EQ)
15000 __ cbzw($op1$$Register, *L);
15001 else
15002 __ cbnzw($op1$$Register, *L);
15003 %}
15004 ins_pipe(pipe_cmp_branch);
15005 %}
15006
15007 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15008 match(If cmp (CmpP (DecodeN oop) zero));
15009 effect(USE labl);
15010
15011 ins_cost(BRANCH_COST);
15012 format %{ "cb$cmp $oop, $labl" %}
15013 ins_encode %{
15014 Label* L = $labl$$label;
15015 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15016 if (cond == Assembler::EQ)
15017 __ cbzw($oop$$Register, *L);
15018 else
15019 __ cbnzw($oop$$Register, *L);
15020 %}
15021 ins_pipe(pipe_cmp_branch);
15022 %}
15023
15024 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15025 match(If cmp (CmpU op1 op2));
15026 effect(USE labl);
15027
15028 ins_cost(BRANCH_COST);
15029 format %{ "cbw$cmp $op1, $labl" %}
15030 ins_encode %{
15031 Label* L = $labl$$label;
15032 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15033 if (cond == Assembler::EQ || cond == Assembler::LS) {
15034 __ cbzw($op1$$Register, *L);
15035 } else {
15036 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15037 __ cbnzw($op1$$Register, *L);
15038 }
15039 %}
15040 ins_pipe(pipe_cmp_branch);
15041 %}
15042
15043 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15044 match(If cmp (CmpUL op1 op2));
15045 effect(USE labl);
15046
15047 ins_cost(BRANCH_COST);
15048 format %{ "cb$cmp $op1, $labl" %}
15049 ins_encode %{
15050 Label* L = $labl$$label;
15051 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15052 if (cond == Assembler::EQ || cond == Assembler::LS) {
15053 __ cbz($op1$$Register, *L);
15054 } else {
15055 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15056 __ cbnz($op1$$Register, *L);
15057 }
15058 %}
15059 ins_pipe(pipe_cmp_branch);
15060 %}
15061
15062 // Test bit and Branch
15063
15064 // Patterns for short (< 32KiB) variants
15065 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15066 match(If cmp (CmpL op1 op2));
15067 effect(USE labl);
15068
15069 ins_cost(BRANCH_COST);
15070 format %{ "cb$cmp $op1, $labl # long" %}
15071 ins_encode %{
15072 Label* L = $labl$$label;
15073 Assembler::Condition cond =
15074 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15075 __ tbr(cond, $op1$$Register, 63, *L);
15076 %}
15077 ins_pipe(pipe_cmp_branch);
15078 ins_short_branch(1);
15079 %}
15080
15081 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15082 match(If cmp (CmpI op1 op2));
15083 effect(USE labl);
15084
15085 ins_cost(BRANCH_COST);
15086 format %{ "cb$cmp $op1, $labl # int" %}
15087 ins_encode %{
15088 Label* L = $labl$$label;
15089 Assembler::Condition cond =
15090 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15091 __ tbr(cond, $op1$$Register, 31, *L);
15092 %}
15093 ins_pipe(pipe_cmp_branch);
15094 ins_short_branch(1);
15095 %}
15096
15097 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15098 match(If cmp (CmpL (AndL op1 op2) op3));
15099 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15100 effect(USE labl);
15101
15102 ins_cost(BRANCH_COST);
15103 format %{ "tb$cmp $op1, $op2, $labl" %}
15104 ins_encode %{
15105 Label* L = $labl$$label;
15106 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15107 int bit = exact_log2_long($op2$$constant);
15108 __ tbr(cond, $op1$$Register, bit, *L);
15109 %}
15110 ins_pipe(pipe_cmp_branch);
15111 ins_short_branch(1);
15112 %}
15113
15114 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15115 match(If cmp (CmpI (AndI op1 op2) op3));
15116 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15117 effect(USE labl);
15118
15119 ins_cost(BRANCH_COST);
15120 format %{ "tb$cmp $op1, $op2, $labl" %}
15121 ins_encode %{
15122 Label* L = $labl$$label;
15123 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15124 int bit = exact_log2((juint)$op2$$constant);
15125 __ tbr(cond, $op1$$Register, bit, *L);
15126 %}
15127 ins_pipe(pipe_cmp_branch);
15128 ins_short_branch(1);
15129 %}
15130
15131 // And far variants
15132 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15133 match(If cmp (CmpL op1 op2));
15134 effect(USE labl);
15135
15136 ins_cost(BRANCH_COST);
15137 format %{ "cb$cmp $op1, $labl # long" %}
15138 ins_encode %{
15139 Label* L = $labl$$label;
15140 Assembler::Condition cond =
15141 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15142 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15143 %}
15144 ins_pipe(pipe_cmp_branch);
15145 %}
15146
15147 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15148 match(If cmp (CmpI op1 op2));
15149 effect(USE labl);
15150
15151 ins_cost(BRANCH_COST);
15152 format %{ "cb$cmp $op1, $labl # int" %}
15153 ins_encode %{
15154 Label* L = $labl$$label;
15155 Assembler::Condition cond =
15156 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15157 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15158 %}
15159 ins_pipe(pipe_cmp_branch);
15160 %}
15161
15162 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15163 match(If cmp (CmpL (AndL op1 op2) op3));
15164 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15165 effect(USE labl);
15166
15167 ins_cost(BRANCH_COST);
15168 format %{ "tb$cmp $op1, $op2, $labl" %}
15169 ins_encode %{
15170 Label* L = $labl$$label;
15171 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15172 int bit = exact_log2_long($op2$$constant);
15173 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15174 %}
15175 ins_pipe(pipe_cmp_branch);
15176 %}
15177
15178 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15179 match(If cmp (CmpI (AndI op1 op2) op3));
15180 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15181 effect(USE labl);
15182
15183 ins_cost(BRANCH_COST);
15184 format %{ "tb$cmp $op1, $op2, $labl" %}
15185 ins_encode %{
15186 Label* L = $labl$$label;
15187 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15188 int bit = exact_log2((juint)$op2$$constant);
15189 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15190 %}
15191 ins_pipe(pipe_cmp_branch);
15192 %}
15193
15194 // Test bits
15195
15196 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15197 match(Set cr (CmpL (AndL op1 op2) op3));
15198 predicate(Assembler::operand_valid_for_logical_immediate
15199 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15200
15201 ins_cost(INSN_COST);
15202 format %{ "tst $op1, $op2 # long" %}
15203 ins_encode %{
15204 __ tst($op1$$Register, $op2$$constant);
15205 %}
15206 ins_pipe(ialu_reg_reg);
15207 %}
15208
15209 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15210 match(Set cr (CmpI (AndI op1 op2) op3));
15211 predicate(Assembler::operand_valid_for_logical_immediate
15212 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15213
15214 ins_cost(INSN_COST);
15215 format %{ "tst $op1, $op2 # int" %}
15216 ins_encode %{
15217 __ tstw($op1$$Register, $op2$$constant);
15218 %}
15219 ins_pipe(ialu_reg_reg);
15220 %}
15221
15222 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15223 match(Set cr (CmpL (AndL op1 op2) op3));
15224
15225 ins_cost(INSN_COST);
15226 format %{ "tst $op1, $op2 # long" %}
15227 ins_encode %{
15228 __ tst($op1$$Register, $op2$$Register);
15229 %}
15230 ins_pipe(ialu_reg_reg);
15231 %}
15232
15233 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15234 match(Set cr (CmpI (AndI op1 op2) op3));
15235
15236 ins_cost(INSN_COST);
15237 format %{ "tstw $op1, $op2 # int" %}
15238 ins_encode %{
15239 __ tstw($op1$$Register, $op2$$Register);
15240 %}
15241 ins_pipe(ialu_reg_reg);
15242 %}
15243
15244
15245 // Conditional Far Branch
15246 // Conditional Far Branch Unsigned
15247 // TODO: fixme
15248
15249 // counted loop end branch near
15250 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15251 %{
15252 match(CountedLoopEnd cmp cr);
15253
15254 effect(USE lbl);
15255
15256 ins_cost(BRANCH_COST);
15257 // short variant.
15258 // ins_short_branch(1);
15259 format %{ "b$cmp $lbl \t// counted loop end" %}
15260
15261 ins_encode(aarch64_enc_br_con(cmp, lbl));
15262
15263 ins_pipe(pipe_branch);
15264 %}
15265
15266 // counted loop end branch far
15267 // TODO: fixme
15268
15269 // ============================================================================
15270 // inlined locking and unlocking
15271
15272 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15273 %{
15274 match(Set cr (FastLock object box));
15275 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15276
15277 ins_cost(5 * INSN_COST);
15278 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15279
15280 ins_encode %{
15281 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15282 %}
15283
15284 ins_pipe(pipe_serial);
15285 %}
15286
15287 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15288 %{
15289 match(Set cr (FastUnlock object box));
15290 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15291
15292 ins_cost(5 * INSN_COST);
15293 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15294
15295 ins_encode %{
15296 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15297 %}
15298
15299 ins_pipe(pipe_serial);
15300 %}
15301
15302 // ============================================================================
15303 // Safepoint Instructions
15304
15305 // TODO
15306 // provide a near and far version of this code
15307
15308 instruct safePoint(rFlagsReg cr, iRegP poll)
15309 %{
15310 match(SafePoint poll);
15311 effect(KILL cr);
15312
15313 format %{
15314 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15315 %}
15316 ins_encode %{
15317 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15318 %}
15319 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15320 %}
15321
15322
15323 // ============================================================================
15324 // Procedure Call/Return Instructions
15325
15326 // Call Java Static Instruction
15327
15328 instruct CallStaticJavaDirect(method meth)
15329 %{
15330 match(CallStaticJava);
15331
15332 effect(USE meth);
15333
15334 ins_cost(CALL_COST);
15335
15336 format %{ "call,static $meth \t// ==> " %}
15337
15338 ins_encode(aarch64_enc_java_static_call(meth),
15339 aarch64_enc_call_epilog);
15340
15341 ins_pipe(pipe_class_call);
15342 %}
15343
15344 // TO HERE
15345
15346 // Call Java Dynamic Instruction
15347 instruct CallDynamicJavaDirect(method meth)
15348 %{
15349 match(CallDynamicJava);
15350
15351 effect(USE meth);
15352
15353 ins_cost(CALL_COST);
15354
15355 format %{ "CALL,dynamic $meth \t// ==> " %}
15356
15357 ins_encode(aarch64_enc_java_dynamic_call(meth),
15358 aarch64_enc_call_epilog);
15359
15360 ins_pipe(pipe_class_call);
15361 %}
15362
15363 // Call Runtime Instruction
15364
15365 instruct CallRuntimeDirect(method meth)
15366 %{
15367 match(CallRuntime);
15368
15369 effect(USE meth);
15370
15371 ins_cost(CALL_COST);
15372
15373 format %{ "CALL, runtime $meth" %}
15374
15375 ins_encode( aarch64_enc_java_to_runtime(meth) );
15376
15377 ins_pipe(pipe_class_call);
15378 %}
15379
15380 // Call Runtime Instruction
15381
15382 instruct CallLeafDirect(method meth)
15383 %{
15384 match(CallLeaf);
15385
15386 effect(USE meth);
15387
15388 ins_cost(CALL_COST);
15389
15390 format %{ "CALL, runtime leaf $meth" %}
15391
15392 ins_encode( aarch64_enc_java_to_runtime(meth) );
15393
15394 ins_pipe(pipe_class_call);
15395 %}
15396
15397 // Call Runtime Instruction without safepoint and with vector arguments
15398 instruct CallLeafDirectVector(method meth)
15399 %{
15400 match(CallLeafVector);
15401
15402 effect(USE meth);
15403
15404 ins_cost(CALL_COST);
15405
15406 format %{ "CALL, runtime leaf vector $meth" %}
15407
15408 ins_encode(aarch64_enc_java_to_runtime(meth));
15409
15410 ins_pipe(pipe_class_call);
15411 %}
15412
15413 // Call Runtime Instruction
15414
15415 instruct CallLeafNoFPDirect(method meth)
15416 %{
15417 match(CallLeafNoFP);
15418
15419 effect(USE meth);
15420
15421 ins_cost(CALL_COST);
15422
15423 format %{ "CALL, runtime leaf nofp $meth" %}
15424
15425 ins_encode( aarch64_enc_java_to_runtime(meth) );
15426
15427 ins_pipe(pipe_class_call);
15428 %}
15429
15430 // Tail Call; Jump from runtime stub to Java code.
15431 // Also known as an 'interprocedural jump'.
15432 // Target of jump will eventually return to caller.
15433 // TailJump below removes the return address.
15434 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15435 // emitted just above the TailCall which has reset rfp to the caller state.
15436 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15437 %{
15438 match(TailCall jump_target method_ptr);
15439
15440 ins_cost(CALL_COST);
15441
15442 format %{ "br $jump_target\t# $method_ptr holds method" %}
15443
15444 ins_encode(aarch64_enc_tail_call(jump_target));
15445
15446 ins_pipe(pipe_class_call);
15447 %}
15448
15449 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15450 %{
15451 match(TailJump jump_target ex_oop);
15452
15453 ins_cost(CALL_COST);
15454
15455 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15456
15457 ins_encode(aarch64_enc_tail_jmp(jump_target));
15458
15459 ins_pipe(pipe_class_call);
15460 %}
15461
15462 // Forward exception.
15463 instruct ForwardExceptionjmp()
15464 %{
15465 match(ForwardException);
15466 ins_cost(CALL_COST);
15467
15468 format %{ "b forward_exception_stub" %}
15469 ins_encode %{
15470 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15471 %}
15472 ins_pipe(pipe_class_call);
15473 %}
15474
15475 // Create exception oop: created by stack-crawling runtime code.
15476 // Created exception is now available to this handler, and is setup
15477 // just prior to jumping to this handler. No code emitted.
15478 // TODO check
15479 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15480 instruct CreateException(iRegP_R0 ex_oop)
15481 %{
15482 match(Set ex_oop (CreateEx));
15483
15484 format %{ " -- \t// exception oop; no code emitted" %}
15485
15486 size(0);
15487
15488 ins_encode( /*empty*/ );
15489
15490 ins_pipe(pipe_class_empty);
15491 %}
15492
15493 // Rethrow exception: The exception oop will come in the first
15494 // argument position. Then JUMP (not call) to the rethrow stub code.
15495 instruct RethrowException() %{
15496 match(Rethrow);
15497 ins_cost(CALL_COST);
15498
15499 format %{ "b rethrow_stub" %}
15500
15501 ins_encode( aarch64_enc_rethrow() );
15502
15503 ins_pipe(pipe_class_call);
15504 %}
15505
15506
15507 // Return Instruction
15508 // epilog node loads ret address into lr as part of frame pop
15509 instruct Ret()
15510 %{
15511 match(Return);
15512
15513 format %{ "ret\t// return register" %}
15514
15515 ins_encode( aarch64_enc_ret() );
15516
15517 ins_pipe(pipe_branch);
15518 %}
15519
15520 // Die now.
15521 instruct ShouldNotReachHere() %{
15522 match(Halt);
15523
15524 ins_cost(CALL_COST);
15525 format %{ "ShouldNotReachHere" %}
15526
15527 ins_encode %{
15528 if (is_reachable()) {
15529 const char* str = __ code_string(_halt_reason);
15530 __ stop(str);
15531 }
15532 %}
15533
15534 ins_pipe(pipe_class_default);
15535 %}
15536
15537 // ============================================================================
15538 // Partial Subtype Check
15539 //
15540 // superklass array for an instance of the superklass. Set a hidden
15541 // internal cache on a hit (cache is checked with exposed code in
15542 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15543 // encoding ALSO sets flags.
15544
15545 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15546 %{
15547 match(Set result (PartialSubtypeCheck sub super));
15548 predicate(!UseSecondarySupersTable);
15549 effect(KILL cr, KILL temp);
15550
15551 ins_cost(20 * INSN_COST); // slightly larger than the next version
15552 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15553
15554 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15555
15556 opcode(0x1); // Force zero of result reg on hit
15557
15558 ins_pipe(pipe_class_memory);
15559 %}
15560
15561 // Two versions of partialSubtypeCheck, both used when we need to
15562 // search for a super class in the secondary supers array. The first
15563 // is used when we don't know _a priori_ the class being searched
15564 // for. The second, far more common, is used when we do know: this is
15565 // used for instanceof, checkcast, and any case where C2 can determine
15566 // it by constant propagation.
15567
15568 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15569 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15570 rFlagsReg cr)
15571 %{
15572 match(Set result (PartialSubtypeCheck sub super));
15573 predicate(UseSecondarySupersTable);
15574 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15575
15576 ins_cost(10 * INSN_COST); // slightly larger than the next version
15577 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15578
15579 ins_encode %{
15580 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15581 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15582 $vtemp$$FloatRegister,
15583 $result$$Register, /*L_success*/nullptr);
15584 %}
15585
15586 ins_pipe(pipe_class_memory);
15587 %}
15588
15589 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15590 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15591 rFlagsReg cr)
15592 %{
15593 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15594 predicate(UseSecondarySupersTable);
15595 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15596
15597 ins_cost(5 * INSN_COST); // smaller than the next version
15598 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15599
15600 ins_encode %{
15601 bool success = false;
15602 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15603 if (InlineSecondarySupersTest) {
15604 success =
15605 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15606 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15607 $vtemp$$FloatRegister,
15608 $result$$Register,
15609 super_klass_slot);
15610 } else {
15611 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15612 success = (call != nullptr);
15613 }
15614 if (!success) {
15615 ciEnv::current()->record_failure("CodeCache is full");
15616 return;
15617 }
15618 %}
15619
15620 ins_pipe(pipe_class_memory);
15621 %}
15622
15623 // Intrisics for String.compareTo()
15624
15625 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15626 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15627 %{
15628 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15629 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15630 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15631
15632 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15633 ins_encode %{
15634 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15635 __ string_compare($str1$$Register, $str2$$Register,
15636 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15637 $tmp1$$Register, $tmp2$$Register,
15638 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15639 %}
15640 ins_pipe(pipe_class_memory);
15641 %}
15642
15643 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15644 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15645 %{
15646 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15647 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15648 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15649
15650 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15651 ins_encode %{
15652 __ string_compare($str1$$Register, $str2$$Register,
15653 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15654 $tmp1$$Register, $tmp2$$Register,
15655 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15656 %}
15657 ins_pipe(pipe_class_memory);
15658 %}
15659
15660 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15661 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15662 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15663 %{
15664 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15665 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15666 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15667 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15668
15669 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15670 ins_encode %{
15671 __ string_compare($str1$$Register, $str2$$Register,
15672 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15673 $tmp1$$Register, $tmp2$$Register,
15674 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15675 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15676 %}
15677 ins_pipe(pipe_class_memory);
15678 %}
15679
15680 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15681 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15682 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15683 %{
15684 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15685 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15686 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15687 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15688
15689 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15690 ins_encode %{
15691 __ string_compare($str1$$Register, $str2$$Register,
15692 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15693 $tmp1$$Register, $tmp2$$Register,
15694 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15695 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15696 %}
15697 ins_pipe(pipe_class_memory);
15698 %}
15699
15700 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15701 // these string_compare variants as NEON register type for convenience so that the prototype of
15702 // string_compare can be shared with all variants.
15703
15704 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15705 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15706 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15707 pRegGov_P1 pgtmp2, rFlagsReg cr)
15708 %{
15709 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15710 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15711 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15712 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15713
15714 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15715 ins_encode %{
15716 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15717 __ string_compare($str1$$Register, $str2$$Register,
15718 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15719 $tmp1$$Register, $tmp2$$Register,
15720 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15721 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15722 StrIntrinsicNode::LL);
15723 %}
15724 ins_pipe(pipe_class_memory);
15725 %}
15726
15727 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15728 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15729 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15730 pRegGov_P1 pgtmp2, rFlagsReg cr)
15731 %{
15732 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15733 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15734 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15735 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15736
15737 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15738 ins_encode %{
15739 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15740 __ string_compare($str1$$Register, $str2$$Register,
15741 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15742 $tmp1$$Register, $tmp2$$Register,
15743 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15744 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15745 StrIntrinsicNode::LU);
15746 %}
15747 ins_pipe(pipe_class_memory);
15748 %}
15749
15750 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15751 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15752 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15753 pRegGov_P1 pgtmp2, rFlagsReg cr)
15754 %{
15755 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15756 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15757 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15758 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15759
15760 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15761 ins_encode %{
15762 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15763 __ string_compare($str1$$Register, $str2$$Register,
15764 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15765 $tmp1$$Register, $tmp2$$Register,
15766 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15767 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15768 StrIntrinsicNode::UL);
15769 %}
15770 ins_pipe(pipe_class_memory);
15771 %}
15772
15773 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15774 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15775 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15776 pRegGov_P1 pgtmp2, rFlagsReg cr)
15777 %{
15778 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15779 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15780 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15781 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15782
15783 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15784 ins_encode %{
15785 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15786 __ string_compare($str1$$Register, $str2$$Register,
15787 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15788 $tmp1$$Register, $tmp2$$Register,
15789 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15790 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15791 StrIntrinsicNode::UU);
15792 %}
15793 ins_pipe(pipe_class_memory);
15794 %}
15795
15796 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15797 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15798 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15799 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15800 %{
15801 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15802 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15803 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15804 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15805 TEMP vtmp0, TEMP vtmp1, KILL cr);
15806 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15807 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15808
15809 ins_encode %{
15810 __ string_indexof($str1$$Register, $str2$$Register,
15811 $cnt1$$Register, $cnt2$$Register,
15812 $tmp1$$Register, $tmp2$$Register,
15813 $tmp3$$Register, $tmp4$$Register,
15814 $tmp5$$Register, $tmp6$$Register,
15815 -1, $result$$Register, StrIntrinsicNode::UU);
15816 %}
15817 ins_pipe(pipe_class_memory);
15818 %}
15819
15820 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15821 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15822 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15823 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15824 %{
15825 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15826 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15827 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15828 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15829 TEMP vtmp0, TEMP vtmp1, KILL cr);
15830 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15831 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15832
15833 ins_encode %{
15834 __ string_indexof($str1$$Register, $str2$$Register,
15835 $cnt1$$Register, $cnt2$$Register,
15836 $tmp1$$Register, $tmp2$$Register,
15837 $tmp3$$Register, $tmp4$$Register,
15838 $tmp5$$Register, $tmp6$$Register,
15839 -1, $result$$Register, StrIntrinsicNode::LL);
15840 %}
15841 ins_pipe(pipe_class_memory);
15842 %}
15843
15844 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15845 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15846 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15847 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15848 %{
15849 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15850 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15851 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15852 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15853 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15854 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15855 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15856
15857 ins_encode %{
15858 __ string_indexof($str1$$Register, $str2$$Register,
15859 $cnt1$$Register, $cnt2$$Register,
15860 $tmp1$$Register, $tmp2$$Register,
15861 $tmp3$$Register, $tmp4$$Register,
15862 $tmp5$$Register, $tmp6$$Register,
15863 -1, $result$$Register, StrIntrinsicNode::UL);
15864 %}
15865 ins_pipe(pipe_class_memory);
15866 %}
15867
15868 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15869 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15870 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15871 %{
15872 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15873 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15874 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15875 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15876 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15877 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15878
15879 ins_encode %{
15880 int icnt2 = (int)$int_cnt2$$constant;
15881 __ string_indexof($str1$$Register, $str2$$Register,
15882 $cnt1$$Register, zr,
15883 $tmp1$$Register, $tmp2$$Register,
15884 $tmp3$$Register, $tmp4$$Register, zr, zr,
15885 icnt2, $result$$Register, StrIntrinsicNode::UU);
15886 %}
15887 ins_pipe(pipe_class_memory);
15888 %}
15889
15890 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15891 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15892 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15893 %{
15894 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15895 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15896 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15897 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15898 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15899 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15900
15901 ins_encode %{
15902 int icnt2 = (int)$int_cnt2$$constant;
15903 __ string_indexof($str1$$Register, $str2$$Register,
15904 $cnt1$$Register, zr,
15905 $tmp1$$Register, $tmp2$$Register,
15906 $tmp3$$Register, $tmp4$$Register, zr, zr,
15907 icnt2, $result$$Register, StrIntrinsicNode::LL);
15908 %}
15909 ins_pipe(pipe_class_memory);
15910 %}
15911
15912 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15913 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15914 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15915 %{
15916 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15917 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15918 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15919 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15920 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15921 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15922
15923 ins_encode %{
15924 int icnt2 = (int)$int_cnt2$$constant;
15925 __ string_indexof($str1$$Register, $str2$$Register,
15926 $cnt1$$Register, zr,
15927 $tmp1$$Register, $tmp2$$Register,
15928 $tmp3$$Register, $tmp4$$Register, zr, zr,
15929 icnt2, $result$$Register, StrIntrinsicNode::UL);
15930 %}
15931 ins_pipe(pipe_class_memory);
15932 %}
15933
15934 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15935 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15936 iRegINoSp tmp3, rFlagsReg cr)
15937 %{
15938 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15939 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15940 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15941 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15942
15943 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15944
15945 ins_encode %{
15946 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15947 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15948 $tmp3$$Register);
15949 %}
15950 ins_pipe(pipe_class_memory);
15951 %}
15952
15953 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15954 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15955 iRegINoSp tmp3, rFlagsReg cr)
15956 %{
15957 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15958 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15959 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15960 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15961
15962 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15963
15964 ins_encode %{
15965 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15966 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15967 $tmp3$$Register);
15968 %}
15969 ins_pipe(pipe_class_memory);
15970 %}
15971
15972 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15973 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15974 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15975 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15976 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15977 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15978 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15979 ins_encode %{
15980 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15981 $result$$Register, $ztmp1$$FloatRegister,
15982 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15983 $ptmp$$PRegister, true /* isL */);
15984 %}
15985 ins_pipe(pipe_class_memory);
15986 %}
15987
15988 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15989 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15990 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15991 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15992 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15993 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15994 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15995 ins_encode %{
15996 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15997 $result$$Register, $ztmp1$$FloatRegister,
15998 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15999 $ptmp$$PRegister, false /* isL */);
16000 %}
16001 ins_pipe(pipe_class_memory);
16002 %}
16003
16004 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16005 iRegI_R0 result, rFlagsReg cr)
16006 %{
16007 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16008 match(Set result (StrEquals (Binary str1 str2) cnt));
16009 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16010
16011 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16012 ins_encode %{
16013 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16014 __ string_equals($str1$$Register, $str2$$Register,
16015 $result$$Register, $cnt$$Register);
16016 %}
16017 ins_pipe(pipe_class_memory);
16018 %}
16019
16020 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16021 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16022 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16023 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16024 iRegP_R10 tmp, rFlagsReg cr)
16025 %{
16026 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16027 match(Set result (AryEq ary1 ary2));
16028 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16029 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16030 TEMP vtmp6, TEMP vtmp7, KILL cr);
16031
16032 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16033 ins_encode %{
16034 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16035 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16036 $result$$Register, $tmp$$Register, 1);
16037 if (tpc == nullptr) {
16038 ciEnv::current()->record_failure("CodeCache is full");
16039 return;
16040 }
16041 %}
16042 ins_pipe(pipe_class_memory);
16043 %}
16044
16045 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16046 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16047 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16048 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16049 iRegP_R10 tmp, rFlagsReg cr)
16050 %{
16051 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16052 match(Set result (AryEq ary1 ary2));
16053 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16054 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16055 TEMP vtmp6, TEMP vtmp7, KILL cr);
16056
16057 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16058 ins_encode %{
16059 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16060 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16061 $result$$Register, $tmp$$Register, 2);
16062 if (tpc == nullptr) {
16063 ciEnv::current()->record_failure("CodeCache is full");
16064 return;
16065 }
16066 %}
16067 ins_pipe(pipe_class_memory);
16068 %}
16069
16070 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16071 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16072 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16073 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16074 %{
16075 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16076 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16077 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16078
16079 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16080 ins_encode %{
16081 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16082 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16083 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16084 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16085 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16086 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16087 (BasicType)$basic_type$$constant);
16088 if (tpc == nullptr) {
16089 ciEnv::current()->record_failure("CodeCache is full");
16090 return;
16091 }
16092 %}
16093 ins_pipe(pipe_class_memory);
16094 %}
16095
16096 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16097 %{
16098 match(Set result (CountPositives ary1 len));
16099 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16100 format %{ "count positives byte[] $ary1,$len -> $result" %}
16101 ins_encode %{
16102 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16103 if (tpc == nullptr) {
16104 ciEnv::current()->record_failure("CodeCache is full");
16105 return;
16106 }
16107 %}
16108 ins_pipe( pipe_slow );
16109 %}
16110
16111 // fast char[] to byte[] compression
16112 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16113 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16114 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16115 iRegI_R0 result, rFlagsReg cr)
16116 %{
16117 match(Set result (StrCompressedCopy src (Binary dst len)));
16118 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16119 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16120
16121 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16122 ins_encode %{
16123 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16124 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16125 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16126 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16127 %}
16128 ins_pipe(pipe_slow);
16129 %}
16130
16131 // fast byte[] to char[] inflation
16132 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16133 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16134 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16135 %{
16136 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16137 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16138 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16139 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16140
16141 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16142 ins_encode %{
16143 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16144 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16145 $vtmp2$$FloatRegister, $tmp$$Register);
16146 if (tpc == nullptr) {
16147 ciEnv::current()->record_failure("CodeCache is full");
16148 return;
16149 }
16150 %}
16151 ins_pipe(pipe_class_memory);
16152 %}
16153
16154 // encode char[] to byte[] in ISO_8859_1
16155 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16156 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16157 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16158 iRegI_R0 result, rFlagsReg cr)
16159 %{
16160 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16161 match(Set result (EncodeISOArray src (Binary dst len)));
16162 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16163 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16164
16165 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16166 ins_encode %{
16167 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16168 $result$$Register, false,
16169 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16170 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16171 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16172 %}
16173 ins_pipe(pipe_class_memory);
16174 %}
16175
16176 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16177 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16178 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16179 iRegI_R0 result, rFlagsReg cr)
16180 %{
16181 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16182 match(Set result (EncodeISOArray src (Binary dst len)));
16183 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16184 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16185
16186 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16187 ins_encode %{
16188 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16189 $result$$Register, true,
16190 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16191 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16192 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16193 %}
16194 ins_pipe(pipe_class_memory);
16195 %}
16196
16197 //----------------------------- CompressBits/ExpandBits ------------------------
16198
16199 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16200 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16201 match(Set dst (CompressBits src mask));
16202 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16203 format %{ "mov $tsrc, $src\n\t"
16204 "mov $tmask, $mask\n\t"
16205 "bext $tdst, $tsrc, $tmask\n\t"
16206 "mov $dst, $tdst"
16207 %}
16208 ins_encode %{
16209 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16210 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16211 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16212 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16213 %}
16214 ins_pipe(pipe_slow);
16215 %}
16216
16217 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16218 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16219 match(Set dst (CompressBits (LoadI mem) mask));
16220 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16221 format %{ "ldrs $tsrc, $mem\n\t"
16222 "ldrs $tmask, $mask\n\t"
16223 "bext $tdst, $tsrc, $tmask\n\t"
16224 "mov $dst, $tdst"
16225 %}
16226 ins_encode %{
16227 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16228 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16229 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16230 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16231 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16232 %}
16233 ins_pipe(pipe_slow);
16234 %}
16235
16236 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16237 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16238 match(Set dst (CompressBits src mask));
16239 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16240 format %{ "mov $tsrc, $src\n\t"
16241 "mov $tmask, $mask\n\t"
16242 "bext $tdst, $tsrc, $tmask\n\t"
16243 "mov $dst, $tdst"
16244 %}
16245 ins_encode %{
16246 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16247 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16248 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16249 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16250 %}
16251 ins_pipe(pipe_slow);
16252 %}
16253
16254 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16255 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16256 match(Set dst (CompressBits (LoadL mem) mask));
16257 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16258 format %{ "ldrd $tsrc, $mem\n\t"
16259 "ldrd $tmask, $mask\n\t"
16260 "bext $tdst, $tsrc, $tmask\n\t"
16261 "mov $dst, $tdst"
16262 %}
16263 ins_encode %{
16264 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16265 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16266 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16267 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16268 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16269 %}
16270 ins_pipe(pipe_slow);
16271 %}
16272
16273 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16274 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16275 match(Set dst (ExpandBits src mask));
16276 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16277 format %{ "mov $tsrc, $src\n\t"
16278 "mov $tmask, $mask\n\t"
16279 "bdep $tdst, $tsrc, $tmask\n\t"
16280 "mov $dst, $tdst"
16281 %}
16282 ins_encode %{
16283 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16284 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16285 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16286 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16287 %}
16288 ins_pipe(pipe_slow);
16289 %}
16290
16291 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16292 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16293 match(Set dst (ExpandBits (LoadI mem) mask));
16294 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16295 format %{ "ldrs $tsrc, $mem\n\t"
16296 "ldrs $tmask, $mask\n\t"
16297 "bdep $tdst, $tsrc, $tmask\n\t"
16298 "mov $dst, $tdst"
16299 %}
16300 ins_encode %{
16301 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16302 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16303 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16304 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16305 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16306 %}
16307 ins_pipe(pipe_slow);
16308 %}
16309
16310 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16311 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16312 match(Set dst (ExpandBits src mask));
16313 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16314 format %{ "mov $tsrc, $src\n\t"
16315 "mov $tmask, $mask\n\t"
16316 "bdep $tdst, $tsrc, $tmask\n\t"
16317 "mov $dst, $tdst"
16318 %}
16319 ins_encode %{
16320 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16321 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16322 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16323 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16324 %}
16325 ins_pipe(pipe_slow);
16326 %}
16327
16328
16329 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16330 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16331 match(Set dst (ExpandBits (LoadL mem) mask));
16332 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16333 format %{ "ldrd $tsrc, $mem\n\t"
16334 "ldrd $tmask, $mask\n\t"
16335 "bdep $tdst, $tsrc, $tmask\n\t"
16336 "mov $dst, $tdst"
16337 %}
16338 ins_encode %{
16339 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16340 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16341 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16342 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16343 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16344 %}
16345 ins_pipe(pipe_slow);
16346 %}
16347
16348 //----------------------------- Reinterpret ----------------------------------
16349 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16350 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16351 match(Set dst (ReinterpretHF2S src));
16352 format %{ "reinterpretHF2S $dst, $src" %}
16353 ins_encode %{
16354 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16355 %}
16356 ins_pipe(pipe_slow);
16357 %}
16358
16359 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16360 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16361 match(Set dst (ReinterpretS2HF src));
16362 format %{ "reinterpretS2HF $dst, $src" %}
16363 ins_encode %{
16364 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16365 %}
16366 ins_pipe(pipe_slow);
16367 %}
16368
16369 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16370 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16371 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16372 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16373 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16374 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16375 // can be omitted in this pattern, resulting in -
16376 // fcvt $dst, $src // Convert float to half-precision float
16377 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16378 %{
16379 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16380 format %{ "convF2HFAndS2HF $dst, $src" %}
16381 ins_encode %{
16382 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16383 %}
16384 ins_pipe(pipe_slow);
16385 %}
16386
16387 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16388 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16389 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16390 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16391 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16392 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16393 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16394 // resulting in -
16395 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16396 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16397 %{
16398 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16399 format %{ "convHF2SAndHF2F $dst, $src" %}
16400 ins_encode %{
16401 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16402 %}
16403 ins_pipe(pipe_slow);
16404 %}
16405
16406 // ============================================================================
16407 // This name is KNOWN by the ADLC and cannot be changed.
16408 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16409 // for this guy.
16410 instruct tlsLoadP(thread_RegP dst)
16411 %{
16412 match(Set dst (ThreadLocal));
16413
16414 ins_cost(0);
16415
16416 format %{ " -- \t// $dst=Thread::current(), empty" %}
16417
16418 size(0);
16419
16420 ins_encode( /*empty*/ );
16421
16422 ins_pipe(pipe_class_empty);
16423 %}
16424
16425 //----------PEEPHOLE RULES-----------------------------------------------------
16426 // These must follow all instruction definitions as they use the names
16427 // defined in the instructions definitions.
16428 //
16429 // peepmatch ( root_instr_name [preceding_instruction]* );
16430 //
16431 // peepconstraint %{
16432 // (instruction_number.operand_name relational_op instruction_number.operand_name
16433 // [, ...] );
16434 // // instruction numbers are zero-based using left to right order in peepmatch
16435 //
16436 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16437 // // provide an instruction_number.operand_name for each operand that appears
16438 // // in the replacement instruction's match rule
16439 //
16440 // ---------VM FLAGS---------------------------------------------------------
16441 //
16442 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16443 //
16444 // Each peephole rule is given an identifying number starting with zero and
16445 // increasing by one in the order seen by the parser. An individual peephole
16446 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16447 // on the command-line.
16448 //
16449 // ---------CURRENT LIMITATIONS----------------------------------------------
16450 //
16451 // Only match adjacent instructions in same basic block
16452 // Only equality constraints
16453 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16454 // Only one replacement instruction
16455 //
16456 // ---------EXAMPLE----------------------------------------------------------
16457 //
16458 // // pertinent parts of existing instructions in architecture description
16459 // instruct movI(iRegINoSp dst, iRegI src)
16460 // %{
16461 // match(Set dst (CopyI src));
16462 // %}
16463 //
16464 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16465 // %{
16466 // match(Set dst (AddI dst src));
16467 // effect(KILL cr);
16468 // %}
16469 //
16470 // // Change (inc mov) to lea
16471 // peephole %{
16472 // // increment preceded by register-register move
16473 // peepmatch ( incI_iReg movI );
16474 // // require that the destination register of the increment
16475 // // match the destination register of the move
16476 // peepconstraint ( 0.dst == 1.dst );
16477 // // construct a replacement instruction that sets
16478 // // the destination to ( move's source register + one )
16479 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16480 // %}
16481 //
16482
16483 // Implementation no longer uses movX instructions since
16484 // machine-independent system no longer uses CopyX nodes.
16485 //
16486 // peephole
16487 // %{
16488 // peepmatch (incI_iReg movI);
16489 // peepconstraint (0.dst == 1.dst);
16490 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16491 // %}
16492
16493 // peephole
16494 // %{
16495 // peepmatch (decI_iReg movI);
16496 // peepconstraint (0.dst == 1.dst);
16497 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16498 // %}
16499
16500 // peephole
16501 // %{
16502 // peepmatch (addI_iReg_imm movI);
16503 // peepconstraint (0.dst == 1.dst);
16504 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16505 // %}
16506
16507 // peephole
16508 // %{
16509 // peepmatch (incL_iReg movL);
16510 // peepconstraint (0.dst == 1.dst);
16511 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16512 // %}
16513
16514 // peephole
16515 // %{
16516 // peepmatch (decL_iReg movL);
16517 // peepconstraint (0.dst == 1.dst);
16518 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16519 // %}
16520
16521 // peephole
16522 // %{
16523 // peepmatch (addL_iReg_imm movL);
16524 // peepconstraint (0.dst == 1.dst);
16525 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16526 // %}
16527
16528 // peephole
16529 // %{
16530 // peepmatch (addP_iReg_imm movP);
16531 // peepconstraint (0.dst == 1.dst);
16532 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16533 // %}
16534
16535 // // Change load of spilled value to only a spill
16536 // instruct storeI(memory mem, iRegI src)
16537 // %{
16538 // match(Set mem (StoreI mem src));
16539 // %}
16540 //
16541 // instruct loadI(iRegINoSp dst, memory mem)
16542 // %{
16543 // match(Set dst (LoadI mem));
16544 // %}
16545 //
16546
16547 //----------SMARTSPILL RULES---------------------------------------------------
16548 // These must follow all instruction definitions as they use the names
16549 // defined in the instructions definitions.
16550
16551 // Local Variables:
16552 // mode: c++
16553 // End: